Loading...
Searching...
No Matches
mpe_targetloader.sas
Go to the documentation of this file.
1/**
2 @file
3 @brief Generic loader for tables configured in &mpelib..mpe_tables
4 @details General wrapper for loading tables using ALL load types.
5
6 Called by admin/postdata. The dclib should already be assigned.
7
8 @param [in] dclib= (NOTPROVIDED) The data controller control library
9 @param [in] libds= the library.dataset to load
10 @param [in] now= (%sysfunc(datetime())) Static processed timestamp
11
12 <h4> SAS Macros </h4>
13 @li bitemporal_dataloader.sas
14 @li mf_existvar.sas
15 @li mp_abort.sas
16 @li mp_loadformat.sas
17 @li mp_lockanytable.sas
18
19 @version 9.2
20 @author 4GL Apps Ltd
21 @copyright 4GL Apps Ltd. This code may only be used within Data Controller
22 and may not be re-distributed or re-sold without the express permission of
23 4GL Apps Ltd.
24
25**/
26
27%macro mpe_targetloader(libds= /* library.dataset to LOAD (target) */
28 ,now= %sysfunc(datetime()) /* static processed timestamp */
29 ,etlsource= /* process from whence the data came */
30 ,STAGING_DS= STAGING_DS /* name of staging (work) dataset which should
31 be appended into the target. */
32 ,LOADTARGET=NO /* set to yes to actually load the target */
33 ,CLOSE_VARS= /* provide close vars to override defaults */
34 ,dclib=NOTPROVIDED
35 ,mdebug=0
36 ,dc_dttmtfmt=%sysfunc(datetime())
37 );
38%local lib ds nobs;
39
40/**
41 * if a format catalog (suffix "-FC") we assume the catalog has already been
42 * created by the calling program with a libds of work.fmtextract
43*/
44%let orig_lib=%upcase(%scan(&libds,1,.));
45%let orig_ds=%upcase(%scan(&libds,2,.));
46%let orig_libds=&libds;
47%if %scan(&libds,2,-)=FC %then %do;
48 %let lib=WORK;
49 %let ds=FMTEXTRACT;
50 %let libds=&lib..&ds;
51%end;
52%else %do;
53 %let lib=&orig_lib;
54 %let ds=&orig_ds;
55%end;
56
57%mp_abort(iftrue= (&dclib=NOTPROVIDED)
58 ,mac=&sysmacroname
59 ,msg=%str(dclib=NOTPROVIDED)
60)
61
62/* get table attributes */
63%let nobs=0;
64data work.sumo_config;
65 set &mpelib..mpe_tables;
66 where &dc_dttmtfmt. lt tx_to
67 and libref="&orig_lib"
68 and dsn="&orig_ds";
69 call symputx('LOADTYPE',loadtype,'l');
70 call symputx('BUSKEY',buskey,'l');
71 call symputx('VAR_TXFROM',var_txfrom,'l');
72 call symputx('VAR_TXTO',var_txto,'l');
73 call symputx('VAR_BUSFROM',var_busfrom,'l');
74 call symputx('VAR_BUSTO',var_busto,'l');
75 call symputx('VAR_PROCESSED',VAR_PROCESSED,'l');
76 call symputx('RK_UNDERLYING',RK_UNDERLYING,'l');
77 %if %length(&CLOSE_VARS)=0 %then %do;
78 call symputx('CLOSE_VARS',CLOSE_VARS,'l');
79 %end;
80 call symputx('nobs',_n_,'l');
81 if missing(AUDIT_LIBDS) then AUDIT_LIBDS="&dclib..MPE_AUDIT";
82 call symputx('AUDIT_LIBDS',AUDIT_LIBDS,'l');
83 put (_all_)(=);
84run;
85
86/* check if table is actually configured to load */
87%if &nobs ne 1 %then %do;
88 proc sql;
89 insert into &mpelib..mpe_loads
90 set USER_NM="%mf_getuser()"
91 ,STATUS='FAILED (BAD DS)'
92 ,CSV_DIR=symget('ETLSOURCE')
93 ,PROCESSED_DTTM=&now;
94%end;
95%mp_abort(iftrue= (&nobs=0)
96 ,mac=&sysmacroname
97 ,msg=%str(Table not registered in &mpelib..mpe_tables)
98)
99%mp_abort(iftrue= (&nobs>1)
100 ,mac=&sysmacroname
101 ,msg=%str(Something is very wrong with the PK in &mpelib..mpe_tables)
102)
103
104
105%if &LOADTYPE=TXTEMPORAL %then %do;
106
107 %bitemporal_dataloader(bus_from=,bus_to= /* explicitly empty*/
108 ,tech_from=&VAR_TXFROM
109 ,tech_to = &VAR_TXTO
110 ,base_lib=&lib
111 ,base_dsn=&ds
112 ,append_lib=WORK
113 ,append_dsn=&STAGING_DS
114 ,high_date='31DEC9999:23:59:59'dt
115 ,PK= &buskey
116 ,ETLSOURCE=&ETLSOURCE
117 ,LOADTYPE=&loadtype
118 ,RK_UNDERLYING=&RK_UNDERLYING
119 ,LOADTARGET=&LOADTARGET
120 ,RK_UPDATE_MAXKEYTABLE=&LOADTARGET
121 ,CLOSE_VARS=&CLOSE_VARS
122 ,processed=&VAR_PROCESSED
123 ,dclib=&dclib
124 ,outds_audit=&AUDIT_LIBDS
125 )
126%end;
127%else %if &loadtype=REPLACE %then %do;
128 %if &LOADTARGET=YES %then %do;
129 %mp_lockanytable(LOCK,lib=&lib,ds=&ds,ref=%str(&etlsource),
130 ctl_ds=&dclib..mpe_lockanytable
131 )
132 data WORK.&STAGING_DS;
133 set WORK.&STAGING_DS;
134 %if %mf_existvar(&libds,&VAR_PROCESSED) %then %do;
135 &VAR_PROCESSED = &now;
136 %end;
137 drop _____DELETE__THIS__RECORD_____;
138 run;
139 proc sql; delete * from &libds;
140 proc append base=&libds data=WORK.&STAGING_DS force nowarn;run;
141
142 %mp_lockanytable(UNLOCK,lib=&lib,ds=&ds,ctl_ds=&dclib..mpe_lockanytable)
143 %end;
144 %else %do;
145 /* is full replace so treat all staged records as new in diff screen */
146 data work.outds_mod work.outds_add ;
147 set work.&staging_ds;
148 output work.outds_add;
149 run;
150 /* previous table will be considered fully deleted */
151 data work.outds_del;
152 set &lib..&ds;
153 run;
154 %end;
155%end;
156%else %if &loadtype=UPDATE %then %do;
157 %bitemporal_dataloader(bus_from=,bus_to=
158 ,tech_from= ,tech_to = /* explicitly empty*/
159 ,base_lib=&lib
160 ,base_dsn=&ds
161 ,append_lib=WORK
162 ,append_dsn=&STAGING_DS
163 ,high_date='31DEC9999:23:59:59'dt
164 ,PK= &buskey
165 ,ETLSOURCE=%superq(etlsource)
166 ,LOADTYPE=UPDATE
167 ,RK_UNDERLYING=&RK_UNDERLYING
168 ,LOADTARGET=&LOADTARGET
169 ,RK_UPDATE_MAXKEYTABLE=&LOADTARGET
170 ,processed=&VAR_PROCESSED
171 ,dclib=&dclib
172 ,outds_audit=&AUDIT_LIBDS
173 )
174%end;
175%else %if &loadtype=FORMAT_CAT %then %do;
176 /**
177 * run mp_formatload
178 * inputs:
179 * - LOADTARGET
180 * - CATALOG
181 * - STAGEDATA
182 * - LOADAUDIT
183 * outputs:
184 * work.outds_add
185 * work.outds_del
186 * work.outds_mod
187 */
188 %mp_loadformat(&orig_libds
189 ,&staging_ds
190 ,loadtarget=&LOADTARGET
191 ,auditlibds=&AUDIT_LIBDS
192 ,locklibds=&dclib..mpe_lockanytable
193 ,delete_col=_____DELETE__THIS__RECORD_____
194 ,outds_add=outds_add
195 ,outds_del=outds_del
196 ,outds_mod=outds_mod
197 ,mdebug=&mdebug
198 )
199%end;
200%else %if &loadtype=BITEMPORAL %then %do;
201 %bitemporal_dataloader(bus_from=&VAR_BUSFROM,bus_to=&VAR_BUSTO
202 ,tech_from=&VAR_TXFROM
203 ,tech_to = &VAR_TXTO
204 ,base_lib=&lib
205 ,base_dsn=&ds
206 ,append_lib=WORK
207 ,append_dsn=&STAGING_DS
208 ,high_date='31DEC9999:23:59:59'dt
209 ,PK= &buskey
210 ,ETLSOURCE=%superq(etlsource)
211 ,LOADTYPE=BITEMPORAL
212 ,RK_UNDERLYING=&RK_UNDERLYING
213 ,LOADTARGET=&LOADTARGET
214 ,RK_UPDATE_MAXKEYTABLE=&LOADTARGET
215 ,CLOSE_VARS=&CLOSE_VARS
216 ,processed=&VAR_PROCESSED
217 ,dclib=&dclib
218 ,outds_audit=&AUDIT_LIBDS
219 )
220%end;
221%else %do;
222 %put %str(WARN)ING: LOADTYPE &LOADTYPE not supported;
223 %let syscc=4;
224 %mp_abort(msg=LOADTYPE &LOADTYPE not supported,mac=mpe_targetloader.sas)
225%end;
226
227%mp_abort(iftrue= (&syscc ne 0)
228 ,mac=&_program
229 ,msg=%str(syscc=&syscc exiting MPE_TARGETLOADER macro)
230)
231
232%mend mpe_targetloader;