Working with files that have lines longer than 32k can be a challenge in SAS due to length restrictions (32,767 for lrecl and data step variable size, or 64k for macro variables). Fortunately, since 9.4M3 there is a new kid on the block - proc lua
- that can handle strings of any size.
The following code example demonstrates how an entire file can be loaded into a single lua string variable, against which string manipulations can be performed, before writing the file back out to disk.
/* setup filenames */
filename inref "/tmp/file.txt";
filename outref "/tmp/replaced.txt";
/* make a dummy file */
data _null_;
file inref;
put 'here is somestring to replace';
put 'here is anotherstring to replace';
run;
proc lua restart;
submit;
-- more bytes = faster processing but will result in
-- trailing blanks if the file is smaller than this
bytes=2
-- read entire file into one lua variable
str=string.rep(' ',bytes)
fid=sas.fopen('inref','I',bytes,'F')
t={}
while (sas.fread(fid)==0) do
rc=sas.fget(fid,str,bytes)
-- use substring to prevent passing of values by reference
t[#t+1]=str:sub(1,bytes)
end
rc=sas.fclose(fid)
bigstr=table.concat(t)
-- perform string manipulation
bigstr=bigstr:gsub('somestring','replaced1')
bigstr=bigstr:gsub('anotherstring','replaced2')
-- write file back out to disk
fid=sas.fopen('outref','O',bytes,'B')
print(fid)
local i=1;
while (i<bigstr:len()) do
str=bigstr:sub(i,i+bytes)
rc=sas.fput(fid,str)
rc=sas.fwrite(fid)
i=i+bytes
end;
rc=sas.fclose(fid)
endsubmit;
run;
/* test result */
data _null_;
infile outref;
input;list;
run;