mpe_targetloader.sas
Go to the documentation of this file.
1 /**
2  @file
3  @brief Generic loader for tables configured in &mpelib..mpe_tables
4  @details General wrapper for loading tables using ALL load types.
5 
6  Called by admin/postdata. The dclib should already be assigned.
7 
8  @param [in] dclib= (NOTPROVIDED) The data controller control library
9  @param [in] libds= the library.dataset to load
10  @param [in] now= (%sysfunc(datetime())) Static processed timestamp
11 
12  <h4> SAS Macros </h4>
13  @li bitemporal_dataloader.sas
14  @li mf_existvar.sas
15  @li mp_abort.sas
16  @li mp_loadformat.sas
17  @li mp_lockanytable.sas
18 
19  @version 9.2
20  @author 4GL Apps Ltd
21  @copyright 4GL Apps Ltd. This code may only be used within Data Controller
22  and may not be re-distributed or re-sold without the express permission of
23  4GL Apps Ltd.
24 
25 **/
26 
27 %macro mpe_targetloader(libds= /* library.dataset to LOAD (target) */
28  ,now= %sysfunc(datetime()) /* static processed timestamp */
29  ,etlsource= /* process from whence the data came */
30  ,STAGING_DS= STAGING_DS /* name of staging (work) dataset which should
31  be appended into the target. */
32  ,LOADTARGET=NO /* set to yes to actually load the target */
33  ,CLOSE_VARS= /* provide close vars to override defaults */
34  ,dclib=NOTPROVIDED
35  ,mdebug=0
36  ,dc_dttmtfmt=%sysfunc(datetime())
37  );
38 %local lib ds nobs;
39 
40 /**
41  * if a format catalog (suffix "-FC") we assume the catalog has already been
42  * created by the calling program with a libds of work.fmtextract
43 */
44 %let orig_lib=%upcase(%scan(&libds,1,.));
45 %let orig_ds=%upcase(%scan(&libds,2,.));
46 %let orig_libds=&libds;
47 %if %scan(&libds,2,-)=FC %then %do;
48  %let lib=WORK;
49  %let ds=FMTEXTRACT;
50  %let libds=&lib..&ds;
51 %end;
52 %else %do;
53  %let lib=&orig_lib;
54  %let ds=&orig_ds;
55 %end;
56 
57 %mp_abort(iftrue= (&dclib=NOTPROVIDED)
58  ,mac=&sysmacroname
59  ,msg=%str(dclib=NOTPROVIDED)
60 )
61 
62 /* get table attributes */
63 %let nobs=0;
64 data work.sumo_config;
65  set &mpelib..mpe_tables;
66  where &dc_dttmtfmt. lt tx_to
67  and libref="&orig_lib"
68  and dsn="&orig_ds";
69  call symputx('LOADTYPE',loadtype,'l');
70  call symputx('BUSKEY',buskey,'l');
71  call symputx('VAR_TXFROM',var_txfrom,'l');
72  call symputx('VAR_TXTO',var_txto,'l');
73  call symputx('VAR_BUSFROM',var_busfrom,'l');
74  call symputx('VAR_BUSTO',var_busto,'l');
75  call symputx('VAR_PROCESSED',VAR_PROCESSED,'l');
76  call symputx('RK_UNDERLYING',RK_UNDERLYING,'l');
77  %if %length(&CLOSE_VARS)=0 %then %do;
78  call symputx('CLOSE_VARS',CLOSE_VARS,'l');
79  %end;
80  call symputx('nobs',_n_,'l');
81  if missing(AUDIT_LIBDS) then AUDIT_LIBDS="&dclib..MPE_AUDIT";
82  call symputx('AUDIT_LIBDS',AUDIT_LIBDS,'l');
83  put (_all_)(=);
84 run;
85 
86 /* check if table is actually configured to load */
87 %if &nobs ne 1 %then %do;
88  proc sql;
89  insert into &mpelib..mpe_loads
90  set USER_NM="%mf_getuser()"
91  ,STATUS='FAILED (BAD DS)'
92  ,CSV_DIR=symget('ETLSOURCE')
93  ,PROCESSED_DTTM=&now;
94 %end;
95 %mp_abort(iftrue= (&nobs=0)
96  ,mac=&sysmacroname
97  ,msg=%str(Table not registered in &mpelib..mpe_tables)
98 )
99 %mp_abort(iftrue= (&nobs>1)
100  ,mac=&sysmacroname
101  ,msg=%str(Something is very wrong with the PK in &mpelib..mpe_tables)
102 )
103 
104 
105 %if &LOADTYPE=TXTEMPORAL %then %do;
106 
107  %bitemporal_dataloader(bus_from=,bus_to= /* explicitly empty*/
108  ,tech_from=&VAR_TXFROM
109  ,tech_to = &VAR_TXTO
110  ,base_lib=&lib
111  ,base_dsn=&ds
112  ,append_lib=WORK
113  ,append_dsn=&STAGING_DS
114  ,high_date='31DEC9999:23:59:59'dt
115  ,PK= &buskey
116  ,ETLSOURCE=&ETLSOURCE
117  ,LOADTYPE=&loadtype
118  ,RK_UNDERLYING=&RK_UNDERLYING
119  ,LOADTARGET=&LOADTARGET
120  ,RK_UPDATE_MAXKEYTABLE=&LOADTARGET
121  ,CLOSE_VARS=&CLOSE_VARS
122  ,processed=&VAR_PROCESSED
123  ,dclib=&dclib
124  ,outds_audit=&AUDIT_LIBDS
125  )
126 %end;
127 %else %if &loadtype=REPLACE %then %do;
128  %if &LOADTARGET=YES %then %do;
129  %mp_lockanytable(LOCK,lib=&lib,ds=&ds,ref=%str(&etlsource),
130  ctl_ds=&dclib..mpe_lockanytable
131  )
132  data WORK.&STAGING_DS;
133  set WORK.&STAGING_DS;
134  %if %mf_existvar(&libds,&VAR_PROCESSED) %then %do;
135  &VAR_PROCESSED = &now;
136  %end;
137  drop _____DELETE__THIS__RECORD_____;
138  run;
139  proc sql; delete * from &libds;
140  proc append base=&libds data=WORK.&STAGING_DS force nowarn;run;
141 
142  %mp_lockanytable(UNLOCK,lib=&lib,ds=&ds,ctl_ds=&dclib..mpe_lockanytable)
143  %end;
144  %else %do;
145  /* is full replace so treat all staged records as new in diff screen */
146  data work.outds_mod work.outds_add ;
147  set work.&staging_ds;
148  output work.outds_add;
149  run;
150  /* previous table will be considered fully deleted */
151  data work.outds_del;
152  set &lib..&ds;
153  run;
154  %end;
155 %end;
156 %else %if &loadtype=UPDATE %then %do;
157  %bitemporal_dataloader(bus_from=,bus_to=
158  ,tech_from= ,tech_to = /* explicitly empty*/
159  ,base_lib=&lib
160  ,base_dsn=&ds
161  ,append_lib=WORK
162  ,append_dsn=&STAGING_DS
163  ,high_date='31DEC9999:23:59:59'dt
164  ,PK= &buskey
165  ,ETLSOURCE=%superq(etlsource)
166  ,LOADTYPE=UPDATE
167  ,RK_UNDERLYING=&RK_UNDERLYING
168  ,LOADTARGET=&LOADTARGET
169  ,RK_UPDATE_MAXKEYTABLE=&LOADTARGET
170  ,processed=&VAR_PROCESSED
171  ,dclib=&dclib
172  ,outds_audit=&AUDIT_LIBDS
173  )
174 %end;
175 %else %if &loadtype=FORMAT_CAT %then %do;
176  /**
177  * run mp_formatload
178  * inputs:
179  * - LOADTARGET
180  * - CATALOG
181  * - STAGEDATA
182  * - LOADAUDIT
183  * outputs:
184  * work.outds_add
185  * work.outds_del
186  * work.outds_mod
187  */
188  %mp_loadformat(&orig_libds
189  ,&staging_ds
190  ,loadtarget=&LOADTARGET
191  ,auditlibds=&AUDIT_LIBDS
192  ,locklibds=&dclib..mpe_lockanytable
193  ,delete_col=_____DELETE__THIS__RECORD_____
194  ,outds_add=outds_add
195  ,outds_del=outds_del
196  ,outds_mod=outds_mod
197  ,mdebug=&mdebug
198  )
199 %end;
200 %else %if &loadtype=BITEMPORAL %then %do;
201  %bitemporal_dataloader(bus_from=&VAR_BUSFROM,bus_to=&VAR_BUSTO
202  ,tech_from=&VAR_TXFROM
203  ,tech_to = &VAR_TXTO
204  ,base_lib=&lib
205  ,base_dsn=&ds
206  ,append_lib=WORK
207  ,append_dsn=&STAGING_DS
208  ,high_date='31DEC9999:23:59:59'dt
209  ,PK= &buskey
210  ,ETLSOURCE=%superq(etlsource)
211  ,LOADTYPE=BITEMPORAL
212  ,RK_UNDERLYING=&RK_UNDERLYING
213  ,LOADTARGET=&LOADTARGET
214  ,RK_UPDATE_MAXKEYTABLE=&LOADTARGET
215  ,CLOSE_VARS=&CLOSE_VARS
216  ,processed=&VAR_PROCESSED
217  ,dclib=&dclib
218  ,outds_audit=&AUDIT_LIBDS
219  )
220 %end;
221 %else %do;
222  %put %str(WARN)ING: LOADTYPE &LOADTYPE not supported;
223  %let syscc=4;
224  %mp_abort(msg=LOADTYPE &LOADTYPE not supported,mac=mpe_targetloader.sas)
225 %end;
226 
227 %mp_abort(iftrue= (&syscc ne 0)
228  ,mac=&_program
229  ,msg=%str(syscc=&syscc exiting MPE_TARGETLOADER macro)
230 )
231 
232 %mend mpe_targetloader;