diff --git a/DEM/cop_dem.py b/DEM/cop_dem.py index 70285a5..17056a8 100755 --- a/DEM/cop_dem.py +++ b/DEM/cop_dem.py @@ -84,11 +84,11 @@ # print (' wget command: ',command) ret=os.system(command) if ret == 0: - print ('retrieved ',rsc) + print (' retrieved ',rsc) if not os.path.exists(dem): address='http://pangea.stanford.edu/sesfs/copernicus/'+dem command = 'wget -qN '+address # print (' wget command: ',command) ret=os.system(command) if ret == 0: - print ('retrieved ',dem) + print (' retrieved ',dem) diff --git a/DEM/createDEMcop.py b/DEM/createDEMcop.py index 0aa7a46..c8b7354 100755 --- a/DEM/createDEMcop.py +++ b/DEM/createDEMcop.py @@ -65,7 +65,7 @@ for j in range(long,long+x): #print ('tile: ',i,j) command = '$PROC_HOME/DEM/cop_dem.py '+str(i)+' '+str(j) - print (command) + print ('\n '+command) ret=os.system(command) # scan the rsc files for width, xstep parameters @@ -98,7 +98,7 @@ heightstr=strings[1].split() xwidth.append(widthstr[1]) xstep.append(xstepstr[1]) - print(demfile,widthstr) + print(' '+demfile,widthstr) widthmin=min(widthmin,int(widthstr[1])) xstepmax=max(xstepmax,float(xstepstr[1])) widthmax=max(widthmax,int(widthstr[1])) @@ -110,7 +110,7 @@ frsc.write(ystepstr[1]+'\n') else: - print ('not found',rsclist[k],'filling with zeros, likely ocean!') + print (' not found',rsclist[k],'filling with zeros, likely ocean!') frsc.write('no_file\n') frsc.write('0\n') frsc.write('0\n') @@ -118,7 +118,7 @@ frsc.write('0\n') ret=os.system('touch '+rsclist[k]) else: - print('not found',rsclist[k],'filling with zeros, likely ocean!') + print(' not found',rsclist[k],'filling with zeros, likely ocean!') frsc.write('no_file\n') frsc.write('0\n') frsc.write('0\n') @@ -140,7 +140,8 @@ os.system('rm updem.rsc') command = '$PROC_HOME/DEM/mosaicDEM demrscparams '+str(x)+' '+str(y) -print (command) +print('\n Mosaicing DEM tiles') +print(' '+command) ret=os.system(command) # and we need an rsc file to go with this fd=open('updem.rsc','w') @@ -170,21 +171,22 @@ left=str(lonmin) right=str(lonmax) command = '$PROC_HOME/DEM/createspecialdem updem updem.rsc '+outdemfile+' '+outdemrscfile+' '+top+' '+bot+' '+left+' '+right+' '+upsamplex+' '+upsampley - -print (command) +print('\n Upsampling DEM') +print(' '+command) ret = os.system(command) # convert geoid-based dem to ellipsoid command='mv '+outdemfile+' '+outdemfile+'.geoid' -print (command) +#print (command) ret=os.system(command) command='mv '+outdemrscfile+' '+outdemrscfile+'.geoid' -print (command) +#print (command) ret=os.system(command) command='$PROC_HOME/DEM/geoid2008_ellipsoid_interpolate '+outdemfile+'.geoid '+outdemrscfile+'.geoid '+outdemfile+' '+outdemrscfile+' $PROC_HOME/DEM/egm2008_geoid_grid' -print (command) +print('\n Referncing to ellipsoid') +print(' '+command) ret=os.system(command) # clean up diff --git a/DEM/createspecialdem b/DEM/createspecialdem new file mode 100644 index 0000000..d186599 Binary files /dev/null and b/DEM/createspecialdem differ diff --git a/DEM/geoid2008_ellipsoid_interpolate b/DEM/geoid2008_ellipsoid_interpolate new file mode 100644 index 0000000..efc8994 Binary files /dev/null and b/DEM/geoid2008_ellipsoid_interpolate differ diff --git a/DEM/mosaicDEM b/DEM/mosaicDEM new file mode 100644 index 0000000..4944771 Binary files /dev/null and b/DEM/mosaicDEM differ diff --git a/EOFrequests/getEOF.py b/EOFrequests/getEOF.py index 1e7e89b..4e9dfa2 100755 --- a/EOFrequests/getEOF.py +++ b/EOFrequests/getEOF.py @@ -11,7 +11,6 @@ import datetime from datetime import date -#print len(sys.argv) ASF=1 # switch to retrieve from ESA or ASF ESA=1-ASF @@ -29,14 +28,9 @@ year=int(acquireddate/10000) month=int((acquireddate-year*10000)/100) day=acquireddate-year*10000-month*100 -#print ('YYYY MM DD ',year, month, day) acq=datetime.date(year,month,day) -#print "acq= ",acq -#print "acq.toordinal= ",acq.toordinal() orbitstartdate=date.fromordinal(acq.toordinal()-1) -#print ("orbitstartdate= ",orbitstartdate) orbitstr=str(orbitstartdate) -#print ('orbitstr ',orbitstr.replace('-','')) # retrieve orbit file from archive if ESA==1: @@ -44,16 +38,8 @@ print (command) proc = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True) (fullquery, err) = proc.communicate() - #print (fullquery) - #print ('fullquery ',str(fullquery)) - #fullquery=str(fullquery) - #print (type(fullquery)) - #print (fullquery.find("href=")) - #print (fullquery.find("EOF\"")) orbitfilelink=fullquery[fullquery.find("href=")+6:fullquery.find("EOF\"")+3] orbitfile=fullquery[fullquery.find(mission):fullquery.find("EOF")+3] - #print ('orbitfilelink ',orbitfilelink) - #print ('orbitfile ',orbitfile) command = "wget -qN "+orbitfilelink print (command) ret = os.system(command) @@ -63,20 +49,14 @@ fcred=open('.credentials','r') username=fcred.readline().rstrip() password=fcred.readline().rstrip() - #print (username) - #print (password) fcred.close() string="_V"+orbitstr.replace('-','') command="wget -q https://s1qc.asf.alaska.edu/aux_poeorb -O - | grep "+mission+"_OPER | grep "+string print (command) proc = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True) (fullquery, err) = proc.communicate() -# print (str(fullquery,'UTF-8')) fullquery=str(fullquery,'UTF-8') -# print (type(fullquery)) orbitfilelink=fullquery[fullquery.find("href=")+6:fullquery.find("EOF\"")+3] -# print (orbitfilelink) command = "wget -qN https://s1qc.asf.alaska.edu/aux_poeorb/"+orbitfilelink+" --user="+username+" --password="+password -# print (command) ret = os.system(command) diff --git a/bin/snaphu b/bin/snaphu new file mode 100644 index 0000000..992548b Binary files /dev/null and b/bin/snaphu differ diff --git a/build_proc b/build_proc index 76cfeb3..1ed27db 100644 --- a/build_proc +++ b/build_proc @@ -94,3 +94,28 @@ gfortran -c processsub.f90 backprojectgpusub.f90 bounds.f90 orbitrangetime.f90 l nvcc -o sentinel_raw_process sentinel_raw_process.o decode_line_memory.o processsub.o backprojectgpusub.o azimuth_compress.o bounds.o orbitrangetime.o latlon.o intp_orbit.o radar_to_xyz.o unitvec.o tcnbasis.o curvature.o cross.o orbithermite.o filelen.o io.o sentineltimingsub.o getburststatevectors.o $PROC_HOME/fft/fftw-3.3.9/.libs/libfftw3f.a -lstdc++ -lgfortran -lgomp cd .. + +cd kp_scripts +cd int +gfortran -o int_simmask int_simmask.f90 +gfortran -o refpointsfromsim refpointsfromsim.f90 +echo 'built int_simmask & refpointsfromsim in kp_scripts/int' +cd .. + +cd ps +gfortran -o cosine_sim_mask cosine_sim_mask.f90 -fopenmp -lrt -lpthread $PROC_HOME/fft/fftw-3.3.9/.libs/libfftw3f.a +echo 'built cosine_sim_mask in kp_scripts/ps' +cd .. + +cd sbas +gfortran -o sbas_unique sbas_unique.f90 svd.f90 -fopenmp -lrt -lpthread +echo 'built sbas_unique in kp_scripts/sbas' +cd .. + +cd util +gfortran -o coverage_mask coverage_mask.f90 +gfortran -o cull_geolist_fromsim cull_geolist_fromsim.f90 -fopenmp -lrt -lpthread +gfortran -o determine_valid determine_valid.f90 -fopenmp -lrt -lpthread +gfortran -o regressheight regressheight.f90 lsq.o +echo 'built coverage_mask, cull_geolist_fromsim, determine_valid, & regressheight in kp_scripts/util' +cd .. diff --git a/gpu_arch b/gpu_arch new file mode 100644 index 0000000..b955385 Binary files /dev/null and b/gpu_arch differ diff --git a/int/crossmul b/int/crossmul new file mode 100644 index 0000000..cb718b4 Binary files /dev/null and b/int/crossmul differ diff --git a/int/filelen.o b/int/filelen.o new file mode 100644 index 0000000..11d00db Binary files /dev/null and b/int/filelen.o differ diff --git a/int/findrefpoints b/int/findrefpoints new file mode 100644 index 0000000..9cbfa36 Binary files /dev/null and b/int/findrefpoints differ diff --git a/int/makecc b/int/makecc new file mode 100644 index 0000000..b6d3c14 Binary files /dev/null and b/int/makecc differ diff --git a/int/refpointsfromsim b/int/refpointsfromsim new file mode 100644 index 0000000..ba53fea Binary files /dev/null and b/int/refpointsfromsim differ diff --git a/kp_scripts/PR_sbas_multimask.py b/kp_scripts/PR_sbas_multimask.py new file mode 100644 index 0000000..3fc796a --- /dev/null +++ b/kp_scripts/PR_sbas_multimask.py @@ -0,0 +1,289 @@ +#!/usr/bin/env python3 +# +# +# PR_sbas_multimask.py -- Given a series of .geo files, generate SBAS time series with basic workflow +# - Each interferogram is treated identically +# - All pixels are used in unwrapping +# - Cosine_similarity index is used to find reference pixels for tropospheric correction & calibration +# - SBAS includes capabilities for consistent and inconsistent coverage +# +# +# INPUTS: + +# FILE REQUIREMENTS: + +# GENERATES: + + +import os +import subprocess +import sys +import time + +##### ----- CHECK USAGE ----- ##### + +subdirname = sys.argv[1] +geolist_file = sys.argv[2] +path_to_GEO = sys.argv[3] +max_tb = sys.argv[4] +max_sb = sys.argv[5] +looksac = sys.argv[6] +looksdn = sys.argv[7] +scenemask = sys.argv[8] +INCONSISTENT = sys.argv[9] +psthresh = sys.argv[10] +simthresh = sys.argv[11] +refthresh = sys.argv[12] +TROPO_CORR = sys.argv[13] + +##### ----- DEFAULT PARAMETERS ----- ##### + + +##### ----- CHECK FOR INPUTS ----- ##### + +print('\n You are reading geolist: '+geolist_file) +print(' You will be processing with the following parameters:') +print(' Max Temporal Baseline: '+max_tb) +print(' Max Spatial Baseline: '+max_sb) +print(' scenemask: '+scenemask) + +if INCONSISTENT == 'N': + print(' Coverage Flag: Consistent') + flag = '0' +else: + print(' Coverage Flag: Inconsistent') + flag = '1' +print(' Tropospheric Correction: '+TROPO_CORR) + +##### ----- RUN SCRIPT ----- ##### +# 1. Make SBAS list given max_tb, max_sb, and geolist +print('\n Making sbaslist and intlist') +command = '$PROC_HOME/kp_scripts/sentinel/sbas_list.py '+geolist_file+'_fullres '+path_to_GEO+' '+'sbaslist_'+max_tb+' '+max_tb+' '+max_sb +print(' '+command) +ret=os.system(command) + +command = '$PROC_HOME/kp_scripts/sbas/intlist_from_sbas.py sbaslist_'+max_tb +print(' '+command) +ret=os.system(command) + +# 2. Read in the params file and elevation.dem.rsc file +with open('params','r') as fpar: + dem = fpar.readline().strip() + rsc = fpar.readline().strip() + +with open(rsc,'r') as frsc: + demwidth = frsc.readline().strip().split()[-1] + demlength = frsc.readline().strip().split()[-1] + +with open('dem.rsc','r') as fe: + unwwidth = fe.readline().strip().split()[-1] + unwlength = fe.readline().strip().split()[-1] + +# Determine the number of slcs from the geolist +with open(geolist_file,'r') as fgeo: + geos = fgeo.readlines() + nslc=str(len(geos)) + for k in range(len(geos)): + geos[k] = geos[k].strip() + +# 3. Make the interferograms +print('\n Making interferograms in sbaslist_'+max_tb) +command = '$PROC_HOME/kp_scripts/sentinel/ps_sbas_igrams.py sbaslist_'+max_tb+' '+rsc+' 1 1 '+demwidth+' '+demlength+' '+looksac+' '+looksdn +print(' '+command) +ret=os.system(command) + +# 4. Make All Similarity Mask for reference pixels, and all similarity masks for each geocoded SLC, if it hasn't already been made +suff = '_'+max_tb +if os.path.isfile('pssim'+suff)==False: + print('\n Making all_similarity_mask for intlist_'+max_tb+' and '+geolist_file) + command = '$PROC_HOME/kp_scripts/ps/cosine_sim_mask intlist_'+max_tb+' '+unwwidth+' pssim'+suff+' '+scenemask+' '+flag+' '+psthresh+' '+simthresh+' '+suff + print(' '+command) + ret=os.system(command) + +print('\n Making similarity masks for individual scene dates, if they have not already been formed') +flag = '0' +if INCONSISTENT == 'Y': + flag = '1' +with open('geolistTEMP','w') as fgeo: + for geo in geos: + date = geo[:8] + if os.path.isfile('cosine_sim/all_similarity_mask_'+date)==False: + fgeo.write(geo+'\n') + else: + command = 'cp cosine_sim/all_similarity_mask_'+date+' .' + ret=os.system(command) + +for geo in geos: + indate = geo[:8] + command = '$PROC_HOME/kp_scripts/sbas/intlist_from_sbas.py sbaslist_'+max_tb+' '+indate + print(' '+command) + ret=os.system(command) + +command = '$PROC_HOME/kp_scripts/ps/make_simmasks_mask_parallel.py '+unwwidth+' geolistTEMP '+scenemask+' '+flag+' '+psthresh+' '+simthresh +print(' '+command) +ret=os.system(command) + + +# 5. Phase Reconstruction for the interferograms using the similarity mask +print('\n Phase Reconstruction with interferogram-specific similarity masks') +# Create the subdirectory if it doesn't exist and move there +if os.path.isdir(subdirname)==False: + os.system('mkdir '+subdirname) + +# Create an intlist for files that have not been reconstructed yet +fintnew = open('intlistTEMP','w') +INTS = [] +with open('intlist_'+max_tb,'r') as fint: + for name in fint.readlines(): + outname = subdirname+'/'+name.strip() + if os.path.isfile(outname)==False: + fintnew.write(name) + INTS.append(name.strip()) +fintnew.close() + +# Run the interpolator +command = '$PROC_HOME/kp_scripts/ps/psfilter_kp.py intlistTEMP'+' '+unwwidth +print(' '+command) +ret=os.system(command) + +for name in INTS: + namein = name+'.interp' + ret=os.system('mv '+namein+' '+subdirname+'/'+name) + +# Remove temporary list to keep things clean +ret=os.system('rm intlistTEMP') + + +# 6. Make ref_locs file +print('\n Finding reference pixels from cosine similarity file') +simfile = 'all_similarity'+suff +scrfile = 'scr'+suff +out_reflocs = 'ref_locs'+suff +out_locs = 'locs'+suff + +command = '$PROC_HOME/kp_scripts/int/refpointsfromsim '+simfile+' '+scrfile+' '+unwwidth+' '+unwlength+' '+out_reflocs+' '+out_locs+' '+refthresh +print(' '+command) +ret=os.system(command) + +# 7. Unwrap Interferograms +print('\n ##### ----- UNWRAPPING INTERFEROGRAMS IN PARALLEL ----- #####') + +# Move to subdirectory and copy over important files +os.chdir(subdirname) + +# Copy intlist and ref_locs to current directory +ret=os.system('cp ../intlist_'+max_tb+' .') +ret=os.system('cp ../sbaslist_'+max_tb+' .') +ret=os.system('cp ../'+out_reflocs+' .') +ret=os.system('cp ../'+scenemask+' .') +ret=os.system('cp ../'+geolist_file+' .') + +# Unwrap +command = '$PROC_HOME/kp_scripts/util/unwrap_parallel.py intlist_'+max_tb+' '+unwwidth+' .. '+INCONSISTENT+' '+scenemask +print(' '+command) +ret=os.system(command) + +# Make an unwlist file +funw = open('unwlist_'+max_tb,'w') +nints = 0 +unwfiles = [] +with open('intlist_'+max_tb,'r') as fint: + for line in fint.readlines(): + outname = line.replace('.int','.unw') + funw.write(outname) + unwfiles.append(outname.strip()) + nints+=1 +funw.close() +nints=str(nints) + +# 8. Tropospheric Correction +if TROPO_CORR == 'Y': + print('\n ##### ----- APPLYING TROPOSPHERIC CORRECTION TO INTERFEROGRAMS ----- #####') + if os.path.isdir('tropo_corr') == False: + ret=os.system('mkdir tropo_corr') + + # Change to directory where you want the files saved + os.chdir('tropo_corr') + + # Copy intlist to current directory and make corresponding unwlist + ret=os.system('cp ../intlist_'+max_tb+' .') + with open('intlist_'+max_tb,'r') as fint: + ints = fint.readlines() + + with open('unwlistTEMP','w') as funw: + for name in ints: + unwfile = '../'+name.replace('.int','.unw') + funw.write(unwfile) + + # Run the regression + command = '$PROC_HOME/kp_scripts/util/regress_igrams.py unwlistTEMP ../../dem '+unwwidth+' '+unwlength+' ../ref_locs_'+max_tb+' '+INCONSISTENT + print(' '+command) + ret=os.system(command) + + # Remove temporary unwlist since it points to the wrong files for further processing + ret=os.system('rm unwlistTEMP') + + ret=os.chdir('..') + +# 9. SBAS Setup +print('\n ##### ----- MAKING SBAS SETUP FILES ----- #####') +command = '$PROC_HOME/kp_scripts/sbas/sbas_setup.py sbaslist_'+max_tb+' '+geolist_file+' _'+max_tb +print(' '+command) +ret=os.system(command) + +# Determine the number of slcs from the geolist +with open(geolist_file,'r') as fgeo: + nslc=str(len(fgeo.readlines())) + + + +# 10. Run SBAS +print('\n ##### ----- RUNNING SBAS ON ORIGINAL INTERFEROGRAMS ----- #####') +suffix = '_'+max_tb +unwlistin = 'unwlist'+suffix +Tmfile = 'Tm'+suffix+'.out' +timedeltafile = 'timedeltas'+suffix+'.out' +deltimefile = 'deltime'+suffix+'.out' +reflocs_file = 'ref_locs'+suffix +auxdir = '../' +if TROPO_CORR == 'Y': + unwlistin = '../'+unwlistin + Tmfile = '../'+Tmfile + timedeltafile = '../'+timedeltafile + deltimefile = '../'+deltimefile + reflocs_file = '../'+reflocs_file + scenemask = '../'+scenemask + ret=os.chdir('tropo_corr') + + auxdir = '../../' + +# Temporarily move auxiliary files (.amp and (possible) .mask) to current directory +for unw in unwfiles: + command = 'mv '+auxdir+unw.replace('.unw','.amp')+' .' + ret = os.system(command) + if INCONSISTENT=='Y': + command = 'mv '+auxdir+unw.replace('.unw','.mask')+' .' + ret=os.system(command) + +# Run the SBAS script +command = '$PROC_HOME/kp_scripts/sbas/sbas_unique '+unwlistin+' '+nints+' '+nslc+' '+unwwidth+' '+Tmfile+' '+timedeltafile+' '+deltimefile+' '+reflocs_file+' '+scenemask+' '+INCONSISTENT +print(' '+command) +ret=os.system(command) + +# Move files back +os.system('mv *.amp '+auxdir) +if INCONSISTENT=='Y': + os.system('mv *.mask '+auxdir) + +# Move back to SBAS directory +if TROPO_CORR == 'Y': + os.chdir('..') + +os.chdir('..') + +print('\n !!!!! ----- PHASE RECONSTRUCTION WITH SINGLE SIMILARITY MASK & SBAS WORKFLOW COMPLETE ----- !!!!!') + +sys.exit(0) + + + diff --git a/kp_scripts/PR_sbas_singlemask.py b/kp_scripts/PR_sbas_singlemask.py new file mode 100644 index 0000000..35125ff --- /dev/null +++ b/kp_scripts/PR_sbas_singlemask.py @@ -0,0 +1,257 @@ +#!/usr/bin/env python3 +# +# +# PR_singlemask.py -- Given a series of .geo files, generate SBAS time series with basic workflow +# - Each interferogram is treated identically +# - All pixels are used in unwrapping +# - Cosine_similarity index is used to find reference pixels for tropospheric correction & calibration +# - SBAS includes capabilities for consistent and inconsistent coverage +# +# +# INPUTS: + +# FILE REQUIREMENTS: + +# GENERATES: + + +import os +import subprocess +import sys +import time + +##### ----- CHECK USAGE ----- ##### + +subdirname = sys.argv[1] +geolist_file = sys.argv[2] +path_to_GEO = sys.argv[3] +max_tb = sys.argv[4] +max_sb = sys.argv[5] +looksac = sys.argv[6] +looksdn = sys.argv[7] +scenemask = sys.argv[8] +INCONSISTENT = sys.argv[9] +psthresh = sys.argv[10] +simthresh = sys.argv[11] +refthresh = sys.argv[12] +TROPO_CORR = sys.argv[13] + +##### ----- DEFAULT PARAMETERS ----- ##### + + +##### ----- CHECK FOR INPUTS ----- ##### + +print('\n You are reading geolist: '+geolist_file) +print(' You will be processing with the following parameters:') +print(' Max Temporal Baseline: '+max_tb) +print(' Max Spatial Baseline: '+max_sb) +print(' scenemask: '+scenemask) + +if INCONSISTENT == 'N': + print(' Coverage Flag: Consistent') + flag = '0' +else: + print(' Coverage Flag: Inconsistent') + flag = '1' +print(' Tropospheric Correction: '+TROPO_CORR) + +##### ----- RUN SCRIPT ----- ##### +# 1. Make SBAS list given max_tb, max_sb, and geolist +print('\n Making sbaslist and intlist') +command = '$PROC_HOME/kp_scripts/sentinel/sbas_list.py '+geolist_file+'_fullres '+path_to_GEO+' '+'sbaslist_'+max_tb+' '+max_tb+' '+max_sb +print(' '+command) +ret=os.system(command) + +command = '$PROC_HOME/kp_scripts/sbas/intlist_from_sbas.py sbaslist_'+max_tb +print(' '+command) +ret=os.system(command) + +# 2. Read in the params file and elevation.dem.rsc file +with open('params','r') as fpar: + dem = fpar.readline().strip() + rsc = fpar.readline().strip() + +with open(rsc,'r') as frsc: + demwidth = frsc.readline().strip().split()[-1] + demlength = frsc.readline().strip().split()[-1] + +with open('dem.rsc','r') as fe: + unwwidth = fe.readline().strip().split()[-1] + unwlength = fe.readline().strip().split()[-1] + +# 3. Make the interferograms +print('\n Making interferograms in sbaslist_'+max_tb) +command = '$PROC_HOME/kp_scripts/sentinel/ps_sbas_igrams.py sbaslist_'+max_tb+' '+rsc+' 1 1 '+demwidth+' '+demlength+' '+looksac+' '+looksdn +print(' '+command) +ret=os.system(command) + +# 4. Make All Similarity Mask +suff = '_'+max_tb +print('\n Making all_similarity_mask for intlist_'+max_tb+' and '+geolist_file) +command = '$PROC_HOME/kp_scripts/ps/cosine_sim_mask intlist_'+max_tb+' '+unwwidth+' pssim'+suff+' '+scenemask+' '+flag+' '+psthresh+' '+simthresh+' '+suff +print(' '+command) +ret=os.system(command) + +# 5. Phase Reconstruction for the interferograms using the similarity mask +print('\n Phase Reconstruction with all_similarity_mask_'+max_tb) +# Create the subdirectory if it doesn't exist and move there +if os.path.isdir(subdirname)==False: + os.system('mkdir '+subdirname) + +# Create an intlist for files that have not been reconstructed yet +fintnew = open('intlistTEMP','w') +INTS = [] +with open('intlist_'+max_tb,'r') as fint: + for name in fint.readlines(): + outname = subdirname+'/'+name.strip() + if os.path.isfile(outname)==False: + fintnew.write(name) + INTS.append(name.strip()) +fintnew.close() + +# Run the interpolator +command = '$PROC_HOME/kp_scripts/ps/psfilter_hz.py intlistTEMP'+' '+unwwidth+' all_similarity_mask_'+max_tb +print(' '+command) +ret=os.system(command) + +for name in INTS: + namein = name+'.interp' + ret=os.system('mv '+namein+' '+subdirname+'/'+name) + +# Remove temporary list to keep things clean +ret=os.system('rm intlistTEMP') + + +# 6. Make ref_locs file +print('\n Finding reference pixels from cosine similarity file') +simfile = 'all_similarity'+suff +scrfile = 'scr'+suff +out_reflocs = 'ref_locs'+suff +out_locs = 'locs'+suff + +command = '$PROC_HOME/kp_scripts/int/refpointsfromsim '+simfile+' '+scrfile+' '+unwwidth+' '+unwlength+' '+out_reflocs+' '+out_locs+' '+refthresh +print(' '+command) +ret=os.system(command) + +# 7. Unwrap Interferograms +print('\n ##### ----- UNWRAPPING INTERFEROGRAMS IN PARALLEL ----- #####') + +# Move to subdirectory and copy over important files +os.chdir(subdirname) + +# Copy intlist and ref_locs to current directory +ret=os.system('cp ../intlist_'+max_tb+' .') +ret=os.system('cp ../sbaslist_'+max_tb+' .') +ret=os.system('cp ../'+out_reflocs+' .') +ret=os.system('cp ../'+scenemask+' .') +ret=os.system('cp ../'+geolist_file+' .') + +# Unwrap +command = '$PROC_HOME/kp_scripts/util/unwrap_parallel.py intlist_'+max_tb+' '+unwwidth+' .. '+INCONSISTENT+' '+scenemask +print(' '+command) +ret=os.system(command) + +# Make an unwlist file +funw = open('unwlist_'+max_tb,'w') +nints = 0 +unwfiles = [] +with open('intlist_'+max_tb,'r') as fint: + for line in fint.readlines(): + outname = line.replace('.int','.unw') + funw.write(outname) + unwfiles.append(outname.strip()) + nints+=1 +funw.close() +nints=str(nints) + +# 8. Tropospheric Correction +if TROPO_CORR == 'Y': + print('\n ##### ----- APPLYING TROPOSPHERIC CORRECTION TO INTERFEROGRAMS ----- #####') + if os.path.isdir('tropo_corr') == False: + ret=os.system('mkdir tropo_corr') + + # Change to directory where you want the files saved + os.chdir('tropo_corr') + + # Copy intlist to current directory and make corresponding unwlist + ret=os.system('cp ../intlist_'+max_tb+' .') + with open('intlist_'+max_tb,'r') as fint: + ints = fint.readlines() + + with open('unwlistTEMP','w') as funw: + for name in ints: + unwfile = '../'+name.replace('.int','.unw') + funw.write(unwfile) + + # Run the regression + command = '$PROC_HOME/kp_scripts/util/regress_igrams.py unwlistTEMP ../../dem '+unwwidth+' '+unwlength+' ../ref_locs_'+max_tb+' '+INCONSISTENT + print(' '+command) + ret=os.system(command) + + # Remove temporary unwlist since it points to the wrong files for further processing + ret=os.system('rm unwlistTEMP') + + ret=os.chdir('..') + +# 9. SBAS Setup +print('\n ##### ----- MAKING SBAS SETUP FILES ----- #####') +command = '$PROC_HOME/kp_scripts/sbas/sbas_setup.py sbaslist_'+max_tb+' '+geolist_file+' _'+max_tb +print(' '+command) +ret=os.system(command) + +# Determine the number of slcs from the geolist +with open(geolist_file,'r') as fgeo: + nslc=str(len(fgeo.readlines())) + + + +# 10. Run SBAS +print('\n ##### ----- RUNNING SBAS ON ORIGINAL INTERFEROGRAMS ----- #####') +suffix = '_'+max_tb +unwlistin = 'unwlist'+suffix +Tmfile = 'Tm'+suffix+'.out' +timedeltafile = 'timedeltas'+suffix+'.out' +deltimefile = 'deltime'+suffix+'.out' +reflocs_file = 'ref_locs'+suffix +auxdir = '../' +if TROPO_CORR == 'Y': + unwlistin = '../'+unwlistin + Tmfile = '../'+Tmfile + timedeltafile = '../'+timedeltafile + deltimefile = '../'+deltimefile + reflocs_file = '../'+reflocs_file + scenemask = '../'+scenemask + ret=os.chdir('tropo_corr') + + auxdir = '../../' + +# Temporarily move auxiliary files (.amp and (possible) .mask) to current directory +for unw in unwfiles: + command = 'mv '+auxdir+unw.replace('.unw','.amp')+' .' + ret = os.system(command) + if INCONSISTENT=='Y': + command = 'mv '+auxdir+unw.replace('.unw','.mask')+' .' + ret=os.system(command) + +# Run the SBAS script +command = '$PROC_HOME/kp_scripts/sbas/sbas_unique '+unwlistin+' '+nints+' '+nslc+' '+unwwidth+' '+Tmfile+' '+timedeltafile+' '+deltimefile+' '+reflocs_file+' '+scenemask+' '+INCONSISTENT +print(' '+command) +ret=os.system(command) + +# Move files back +os.system('mv *.amp '+auxdir) +if INCONSISTENT=='Y': + os.system('mv *.mask '+auxdir) + +# Move back to SBAS directory +if TROPO_CORR == 'Y': + os.chdir('..') + +os.chdir('..') + +print('\n !!!!! ----- PHASE RECONSTRUCTION WITH SINGLE SIMILARITY MASK & SBAS WORKFLOW COMPLETE ----- !!!!!') + +sys.exit(0) + + + diff --git a/kp_scripts/asfdata/asfwget_notebook.py b/kp_scripts/asfdata/asfwget_notebook.py new file mode 100644 index 0000000..7db6770 --- /dev/null +++ b/kp_scripts/asfdata/asfwget_notebook.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +# +# +# asfwget - download set of asf raw sentinel products +# +# command template: +# "https://datapool.asf.alaska.edu/RAW/SA/S1A_IW_RAW__0SDV_20200209T043049_20200209T043122_031171_039571_0730.zip" +# + +import os +import sys +import subprocess + +##### ----- DEFUALT PARAMETERS ----- ##### +scenefile = 'scenelist' +redownloadflag = 'N' +npar=32 # number to download in parallel + +if len(sys.argv) < 3: + print ("usage: asfwget_notebook_kp.py username password \n"," scenelist contains granule names starting with S1(A,B)") + sys.exit(0) + +##### ----- GET PARAMETERS ----- ##### +username=sys.argv[1] +password=sys.argv[2] + +if len(sys.argv)>3: + scenefile=sys.argv[3] +if len(sys.argv)>4: + redownloadflag = sys.argv[4] +if len(sys.argv)>5: + npar=sys.argv[5] + npar = int(npar) + +##### ----- OPEN FILELIST with FILES TO DOWNLOAD ----- ##### +fscenes=open(scenefile,'r') +scenelist=fscenes.readlines() +fscenes.close() + +##### ----- OPEN REFERENCE FILE (FILES THAT HAVE ALREADY BEEN DOWNLOADED ----- ##### +downloadfile = 'asf_downloaded' +existing = [] +if os.path.isfile(downloadfile): + with open(downloadfile,'r') as flist: + existing=flist.readlines() + flist=open(downloadfile,'a') +else: + flist=open(downloadfile,'w') + +##### ----- DOWNLOAD THE FILES IN SCENELIST ----- ##### +AB='' +num=0 +command=[] +for scene in scenelist: + + # check if file has already been downloaded + if redownloadflag == 'N': + if scene.rstrip()+'\n' in existing: + continue + + # if redownload flag = yes or the file doesn't exist, download the file + if len(scene)>1: + # Check to make sure the file doesn't still exist + filename = scene.replace('-RAW','').strip()+'.zip' + if os.path.isfile(filename): + continue + + if scene.find('S1A')>=0: + AB='A' + if scene.find('S1B')>=0: + AB='B' + + print ('Downloading scene: ',scene.rstrip()) + command.append(subprocess.Popen(['wget','-qN','--user='+username,'--password='+password,'https://datapool.asf.alaska.edu/RAW/S'+AB+'/'+scene.replace('-RAW','').rstrip()+'.zip'])) + num=num+1 + flist.write(scene.rstrip()+'\n') + if num % npar == 0: + for i in range(num-npar,num): + command[i].wait() + +for i in range(num): + command[i].wait() + +flist.close() +print('\n ASF Download complete!\n') + +sys.exit() diff --git a/kp_scripts/cull_geolist_fromsim.py b/kp_scripts/cull_geolist_fromsim.py new file mode 100644 index 0000000..cba72f2 --- /dev/null +++ b/kp_scripts/cull_geolist_fromsim.py @@ -0,0 +1,203 @@ +#!/usr/bin/env python3 +# +# +# cull_geolist_fromsim.py -- Determine if your list of geofiles contains acquisitions with bad data and cull from list +# 1. Make sbaslist and intlist +# 2. Form interferograms if they do not yet exist +# 3. Make all similarity masks +# 4. Find days with insufficient percentage of PS pixels relative to study area and based on input thresh +# 5. Make the final geolist +# 6. Remake the daily all_similarity_mask for each .geo file +# +# INPUTS: + # geolist_file: list of full_resolution *.geo files + # path_to_geo: full path to the .geo files, (does not include final /) + # scenemask: name of scenemask file containing potentially-valid pixels in the study area + # looksac: Number of looks along x direction + # looksdn: Number of looks along y direction + # inconsistent_flag: Indicates whether the geolist contains files that don't have matching coverage. + # 'N'=all files have same coverage, 'Y'=some files have different coverage + + # temporalbaseline: maximum temporal baseline for forming interferograms (default=90) + # spatialbaseline: maximum spatial baseline for forming interferograms (default=1000) + # psthresh: Threshold for determining PS pixels from mle estimator (default=2) + # simthresh: Threshold for determining additional PS pixels from cosine similarity index + # (default = 0.5) + # prct_thresh: Threshold for deciding whether to cull an SLC from the geolist if its coverage is + # insufficient relative to the valid coverage indicated in scenemask + # (default = 0.1, i.e. 10%) +# FILE REQUIREMENTS: + # params: parameters file in the current directory + # dem.rsc: multilooked dem.rsc file + # *.dem, *.dem.rsc: elevation files that params points to + +# GENERATES: + +import os +import subprocess +import sys +import time + +if len(sys.argv)<7: + print('Usage: cull_geolist_fromsim.py geolist_file path_to_geo scenemask looksac looksdn INCONSISTENT_FLAG ') + sys.exit(0) + +##### ----- REQUIRED PARAMETERS ----- ##### +geolist_file = sys.argv[1] +path_to_geo = sys.argv[2] +scenemask = sys.argv[3] +looksac = sys.argv[4] +looksdn = sys.argv[5] +inconsistent_flag = sys.argv[6] + +##### ----- DEFAULT PARAMETERS ----- ##### +max_tb = '90' +max_sb = '1000' +psthresh = '2' +simthresh = '0.5' +prct_thresh = '0.1' + +##### ----- CHECK FOR INPUTS ----- ##### +args = len(sys.argv) + +if args>7: + max_tb = str(sys.argv[7]) + +if args>8: + max_sb = str(sys.argv[8]) + +if args>9: + psthresh = str(sys.argv[9]) + +if args>10: + simthresh = str(sys.argv[10]) + +if args>11: + prct_thresh = str(sys.argv[11]) + + + +##### ----- RUN SCRIPT ----- ##### +# 1. Determine the multilooked study area size and path to original DEM +with open('dem.rsc','r') as fe: + words=fe.readline() + unwwidth=words.split()[1].strip() + words=fe.readline() + unwlength=words.split()[1].strip() + +with open('params','r') as fparams: + demin = fparams.readline().strip() + rscin = fparams.readline().strip() + +with open(rscin,'r') as fe: + words = fe.readline() + demwidth=words.split()[1].strip() + words=fe.readline() + demlength=words.split()[1].strip() + +# 2. Make SBAS list +print('\n Making SBAS list for Maximum Temporal Baseline = '+max_tb+' days') +command = '$PROC_HOME/kp_scripts/sentinel/sbas_list.py '+geolist_file+'_fullres '+path_to_geo+' sbaslist_'+max_tb+' '+max_tb+' '+max_sb +print(' '+command) +ret=os.system(command) + +# 3. Make the corresponding intlist +command = '$PROC_HOME/kp_scripts/sbas/intlist_from_sbas.py sbaslist_'+max_tb +print(' '+command) +ret=os.system(command) + +# 4. Make the interferograms in the list +print('\n Making interferograms in sbaslist_'+max_tb) +command = '$PROC_HOME/kp_scripts/sentinel/ps_sbas_igrams.py sbaslist_'+max_tb+' '+rscin+' 1 1 '+demwidth+' '+demlength+' '+looksac+' '+looksdn +print(' '+command) +ret=os.system(command) + +# 5. If inconsistent_flag is 'Y', create the interferogram masks +if inconsistent_flag == 'Y': + print('\n Making individual interferogram scene masks') + command = '$PROC_HOME/kp_scripts/int/int_coverage_mask.py intlist_'+max_tb+' '+unwwidth + print(' '+command) + ret=os.system(command) +else: + print('\n All files in geolist are consistent, using single scene mask') + +# 6. Make intlists for each .geo date +print('\n Making intlist for each .geo file') +with open(geolist_file) as fgeo: + geos = fgeo.readlines() + +for geo in geos: + indate = geo[:8] + command = '$PROC_HOME/kp_scripts/sbas/intlist_from_sbas.py sbaslist_'+max_tb+' '+indate + print(' '+command) + ret=os.system(command) + +nslc = len(geos) + +# 6. Generate all_similarity_mask for each .geo file using cosine similarity index +print('\n Making all_similarity_mask for each geocoded slc') +flag = '0' +if inconsistent_flag == 'Y': + flag = '1' +command = '$PROC_HOME/kp_scripts/ps/make_simmasks_mask_parallel.py '+unwwidth+' '+geolist_file+' '+scenemask+' '+flag+' '+psthresh+' '+simthresh +print(' '+command) +ret=os.system(command) + +# 7. Determine if there are SLCs that should be culled from the geolist +print('\n Determining final geolist') +command = '$PROC_HOME/kp_scripts/util/cull_geolist_fromsim '+geolist_file+' '+unwwidth+' '+unwlength+' '+str(nslc)+' '+geolist_file+'_final '+scenemask+' '+prct_thresh +print(' '+command) +ret=os.system(command) + +# 8. Determine if the final geolist is different than the input geolist +rerun = 'N' +with open(geolist_file+'_final','r') as fgeo: + geolist_final = fgeo.readlines() +if len(geolist_final) != nslc: + rerun = 'Y' +nslc = len(geolist_final) + +# 9. If rerun = 'Y' then remake the all_similarity_mask for each SLC in updated geolist +if os.path.isdir('cosine_sim')==False: + ret=os.system('mkdir cosine_sim') +ret=os.system('mv all_similarity* median_sim* pssim* scr* cosine_sim_par* cosine_sim') + +if rerun == 'Y': + print('\n Remaking all_similarity_mask files for each date in final geolist') + with open(geolist_file+'_final_fullres','w') as fgeo: + for geo in geolist_final: + outname = path_to_geo+'/'+geo.strip().replace('_multi','')+'\n' + fgeo.write(outname) + + # Make New SBAS list + command = '$PROC_HOME/kp_scripts/sentinel/sbas_list.py '+geolist_file+'_final_fullres '+path_to_geo+' sbaslist_'+max_tb+' '+max_tb+' '+max_sb + print(' '+command) + ret=os.system(command) + + command = '$PROC_HOME/kp_scripts/sbas/intlist_from_sbas.py sbaslist_'+max_tb + print(' '+command) + ret=os.system(command) + + # Make new intlist for each geolist + for geo in geolist_final: + indate = geo[:8] + command = '$PROC_HOME/kp_scripts/sbas/intlist_from_sbas.py sbaslist_'+max_tb+' '+indate + ret=os.system(command) + + # Recalculate the Similarity Masks + command = '$PROC_HOME/kp_scripts/ps/make_simmasks_mask_parallel.py '+unwwidth+' '+geolist_file+'_final '+scenemask+' '+flag+' '+psthresh+' '+simthresh + print(' '+command) + ret=os.system(command) + + ret=os.system('mv all_similarity* median_sim* pssim* scr* cosine_sim_par* cosine_sim') +else: + print(' No Rerun needed.') + + + + + + + + + diff --git a/kp_scripts/data_arch.py b/kp_scripts/data_arch.py new file mode 100644 index 0000000..53e295c --- /dev/null +++ b/kp_scripts/data_arch.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 +# +# +# data_arch.py -- generate subdirectories for a study area, create DEM, and do file management +# +# +# INPUTS: + # summary_file: file with study area information (default = 'summary_paths' +# GENERATES: + # subdirectories: ascending, descending + # sub-subdirectories: path_[number] + # creates file 'subdirectores' which lists the full paths to each subdirectory + # elevation.dem for study area + # elevation.dem.rsc for study area + # creates 'params' file for DEM files and copies to path_[number] subdirectories + # copies fileids for each path_[number] to scenelist in appropriate subdirectory + # puts extra DEM files into directory DEM + +import os +import subprocess +import sys +import time + +##### ----- DEFAULT PARAMETERS ----- ##### +sum_file = 'summary_paths' +upsample_x = '6' # 30 m dem / 6 = 5m posting (approximate Sentinel-1 resolution in range = 5 m) +upsample_y = '2' # 30m dem / 2 = 15m posting (approximage Sentinel-1 resolution in azimuth = 15 m) + +##### ----- CHECK FOR INPUTS ----- ##### +if len(sys.argv)>=2: + sum_file = str(sys.argv[1]) + +if len(sys.argv)>=3: + upsample_x = str(sys.argv[2]) + +if len(sys.argv)>=4: + upsample_y = str(sys.argv[3]) + +##### ----- RUN SCRIPT ----- ##### +print ( "\n"+"##### ----- Set up data architecture for your study area ----- #####"+"\n") + +# Read the data from summary file +print(' Reading summary file: '+sum_file) +fid = open(sum_file,'r') +f = fid.readlines() +fid.close() + +# Get extents to create the special DEM (upsampled to approx. native posting of the satellite, 5x15 m) +print(' Study Area Extents:') +min_lon = f[0].split(':')[1].replace('\n','') +max_lon = f[1].split(':')[1].replace('\n','') +min_lat = f[2].split(':')[1].replace('\n','') +max_lat = f[3].split(':')[1].replace('\n','') +print(' Minimum Longitude: '+min_lon) +print(' Maximum Longitude: '+max_lon) +print(' Minimum Latitude: '+min_lat) +print(' Maximum Latitude: '+max_lat+'\n') + +# Check to see if elevation.dem/elevation.dem.rsc already exists +dem_flag = 0 # 0=correct DEM does not exist; 1=correct DEM exists +if os.path.isfile('elevation.dem.rsc') == True: + # Check DEM input parameters + input_file = 'DEM_input_params' + if os.path.isfile(input_file) == True: + with open(input_file,'r') as params: + lines = params.readlines() + xmin = lines[0].split(':')[-1].strip() + xmax = lines[1].split(':')[-1].strip() + ymin = lines[2].split(':')[-1].strip() + ymax = lines[3].split(':')[-1].strip() + xup = lines[4].split(':')[-1].strip() + yup = lines[5].split(':')[-1].strip() + # If existing DEM_input_params file matches input parameters to script, set dem_flag = 1 (exists) + if (xmin==min_lon) and (xmax==max_lon) and (ymin==min_lat) and (ymax==max_lat) and (xup==upsample_x) and (yup==upsample_y): + dem_flag = 1 + with open('Update_Files_Flag','w') as flag: + flag.write('UPDATE_EXISTING: N') + else: + with open('Update_Files_Flag','w') as flag: + flag.write('UPDATE_EXISTING: Y') +else: + with open('Update_Files_Flag','w') as flag: + flag.write('UPDATE_EXISTING: N') + + +# If elevation.dem.rsc does not exist, create a DEM +if dem_flag==0: + command = '$PROC_HOME/DEM/createDEMcop.py elevation.dem elevation.dem.rsc '+max_lat+' '+min_lat+' '+min_lon+' '+max_lon+' '+upsample_x+' '+upsample_y + print(' Creating DEM') + print(' Upsampling 30 m DEM by '+upsample_x+' across and '+upsample_y+' down') + print(' '+command) + ret = os.system(command) + print('\n DEM created!') + print(' Cleaning up directory:') + command ='mkdir DEM' + print(' '+command) + ret = os.system(command) + command = 'mv *.geoid dem* no_file latloncoords DEM' + print(' '+command) + ret = os.system(command) + + # create the params file + print('\n DEM file set to elevation.dem') + print(' DEM resource (rsc) file set to elevation.dem.rsc') + print(' Writing params file\n') + with open('params','w') as params: + params.write(os.getcwd()+'/elevation.dem'+'\n') + params.write(os.getcwd()+'/elevation.dem.rsc'+'\n') + + with open('DEM_input_params','w') as demparams: + demparams.write('XMIN: '+min_lon+'\n') + demparams.write('XMAX: '+max_lon+'\n') + demparams.write('YMIN: '+min_lat+'\n') + demparams.write('YMAX: '+max_lat+'\n') + demparams.write('Xupsample: '+upsample_x+'\n') + demparams.write('Yupsample: '+upsample_y+'\n') +else: + print('\n DEM with matching parameters already exists, skipping creation!\n') + +print(' Creating Data Architecture') +# Create data architecture based on the path lines +subdir = open('subdirectories','w') +for i in range(len(f)): + if i<7: + continue + words = f[i].split(',') + direction = words[1] + if os.path.isdir(direction) == False: + command = 'mkdir '+direction + print(' '+command) + ret = os.system(command) + path_num = words[0] + path_dir = direction+'/path_'+path_num + subdir.write(os.getcwd()+'/'+path_dir+'\n') + if os.path.isdir(path_dir) == False: + command = 'mkdir '+path_dir + print(' '+command) + ret = os.system(command) + fileidlist = 'fileids_'+direction+'_'+path_num + if os.path.isfile(fileidlist)==True: + command = 'cp '+fileidlist+' '+path_dir+'/scenelist' + print(' '+command) + ret = os.system(command) + else: + print(' WARNING: Cannot copy '+fileidlist+' to appropriate subdirectory because file does not exist!') + command = 'cp params '+path_dir + print(' '+command) + ret = os.system(command) +subdir.close() + +print('\n Data architecture setup complete!') + + + diff --git a/kp_scripts/determine_consistency.py b/kp_scripts/determine_consistency.py new file mode 100644 index 0000000..d525c4b --- /dev/null +++ b/kp_scripts/determine_consistency.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +# +# +# determine_consistency.py -- Determine if your list of geofiles contains acquisitions with consistent coverage. +# Make a 'geolist_consistent' and a 'geolist_culled' +# Where consistent are the consistent *.geo files with the most temporal coverage +# and culled are a list of inconsistent coverage, where the coverage of each .geo file +# exceeds the input threshold. +# The coverage is determined in reference to the scenemask, where only pixels covered +# by the existing *.geo files are considered valid. +# +# +# INPUTS: + # geolist_file: list of full_resolution *.geo files + # looksac: Number of looks along x direction + # looksdn: Number of looks along y direction + # update_flag: if files need to be recreated, set update_flag = 'Y'. If files do NOT + # need to be recreated, keep update_flag = 'N'. (default = 'N') + # WATER_MASK: if you'd like to create a water mask and intersect it with the scene-level masks + # set to 'Y' (default = N). Note: isn't currently functional + # LOOK_VECTOR: if you'd like to create look vectors for each multilooked .geo file, set to 'Y' + # (default = 'N') + +# FILE REQUIREMENTS: + # params: parameters file in the current directory + +# GENERATES: + # *_multi.geo: multilooked geofiles with _multi tag + # geolist_ml: geolist equivalen to geolist_file, but pointing to the multilooked files + # dem: multilooked DEM in current directory + # dem.rsc: multilooked DEM rsc file in current directory + # *_multi.mask: a mask designating where the acquisition has valid coverage. Note: if WATER_MASK + # is 'Y' (and functional), then this mask includes valid coverage and land intersection + # mask_water: water maks designating where pixels are land (not currently working!) + # *.labels: Label file for look vector + # *.lookvector: Look vector file + +import os +import subprocess +import sys +import time + +if len(sys.argv)<4: + print('Usage: determine_consistency.py geolist_file path_to_geo coverage_thresh consistent_thresh') + sys.exit(0) + +##### ----- REQUIRED PARAMETERS ----- ##### +geolist_file = sys.argv[1] +path_to_geo = sys.argv[2] +coverage_thresh = str(sys.argv[3]) +consistent_thresh = str(sys.argv[4]) + +##### ----- DEFAULT PARAMETERS ----- ##### + +##### ----- CHECK FOR INPUTS ----- ##### + +##### ----- RUN SCRIPT ----- ##### +# 1. Determine the multilooked study area size +with open('dem.rsc','r') as fe: + words=fe.readline() + unwwidth=words.split()[1].strip() + words=fe.readline() + unwlength=words.split()[1].strip() + +# 2. Open the geolist +with open(geolist_file,'r') as fgeo: + geolist = fgeo.readlines() + nslc = str(len(geolist)) + +# 3. Find the combined coverage (potentially-valid pixels) +print('\n ##### ----- Determining Valid Coverage, Quantifying Coverage, and Selecting SLCs ----- #####') +command = '$PROC_HOME/kp_scripts/util/determine_valid '+geolist_file+' '+nslc+' '+unwwidth+' '+unwlength+' '+coverage_thresh +print(' '+command) +ret=os.system(command) + +# 4. Determine if the consistent and culled geolist are the same and write to CONSISTENT_FLAG +with open('geolist_consistent','r') as fgeo: + geolist_consistent = fgeo.readlines() + nslc_consistent = len(geolist_consistent) + +with open('geolist_culled','r') as fgeo: + geolist_culled = fgeo.readlines() + nslc_culled = len(geolist_culled) + +culled_flag = 'N' +consistent_flag = 'Y' +if nslc_consistent == nslc_culled: + if geolist_consistent == geolist_culled: + print('\n The Consistent and Culled Geolists are the same. You only need to process the Consistent') + else: + print('\n The Consistent and Culled Geolists are NOT the same, but have the same number of SLCs') + print(' Defaulting to Consistent geolist.') +elif nslc_culled-nslc_consistent=2: + scenelist_file = str(sys.argv[1]) + +if len(sys.argv)>=3: + update_flag = str(sys.argv[2]) + +print('\n You are reading scene list: '+scenelist_file+'; and update_flag = '+update_flag) + +##### ----- RUN SCRIPT ----- ##### +# 1. Get ASF username and password for data download +print(' You will now be downloading files to the subdirectories for your study area. You may be prompted for you Earthdata username and password. Your password will not be displayed, but it will be stored temporarily in a file only you can access, which will then be deleted. \nNote: If the requested raw data file has already been processed, it will not be downloaded again unless Update_Files_Flag=="Y" AND the file has not already been deleted. If you want to download a newer or more complete file, delete the file from "path_[number]/RAW" before executing the following.') + +if os.path.isfile('.credentials')==False: + print('Earthdata username: '), + EDuser=input() + print('Earthdata pawword (no echo): '), + EDpassword=getpass.getpass() + f=open('.credentials','w') + os.chmod('.credentials',0o600) + f.write(EDuser+'\n') + f.write(EDpassword+'\n') + f.close() +else: + print('\n Already have Earthdata credentials') + creds = open('.credentials','r') + EDuser=creds.readline().rstrip() + EDpassword=creds.readline().rstrip() + creds.close() + +# 2. Download scenes in each subdirectory (path number from summary_paths file) +directory = os.getcwd() +print('\n ##### ----- Downloading L0 data files ----- #####') + +print('\n Downloading RAW files in '+directory) +command = '$PROC_HOME/kp_scripts/asfdata/asfwget_notebook.py '+EDuser+' '+EDpassword+' '+scenelist_file+' '+update_flag+' 32' +print(' $PROC_HOME/kp_scripts/asfdata/asfwget_notebook.py '+EDuser+' '+scenelist_file+' '+update_flag+' 32') +ret = os.system(command) + +print('\n All available files downloaded!') + + + + diff --git a/kp_scripts/generate_geo.py b/kp_scripts/generate_geo.py new file mode 100644 index 0000000..79d4416 --- /dev/null +++ b/kp_scripts/generate_geo.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +# +# +# generate_geo.py -- Process sentinel files to individual geocoded SLCs, then merge same-day L0 scenes +# +# +# INPUTS: + # home: path to study area home, which has params file and .credentials file + # path_to_merged: path to where you want to store the merged (final) .geo files + # update_flag: if files need to be redownloaded/reprocessed, set update_flag = 'Y'. If files do NOT + # need to be redownloaded, keep update_flag = 'N'. (default = 'N') + # REMOVE_ZIP: remove *.zip files after they have been processed (default = 'N') + # REMOVE_SAFE: remove *.SAFE directories after they have been processed (default = 'N') + # REMOVE_ORBTIMING: remove *.orbtiming* files from RAW after L0 files have been merged (default = 'N') + # REMOVE_RAW_GEO: remove intermediate .geo files (pre-merge) after they have been merged (default = 'N') + +# FILE REQUIREMENTS: + # params: parameters file in processing directory (home), to be copied to RAW + # .credentials: file with Earthdata login, in processing directory (home), to be copied to RAW + +# GENERATES: + # /GEO: GEO directory that holds the merge .geo files and .orbtiming files + # [yyyymmdd]_merged.geo: merged *.geo files, in GEO directory. Even when there are not multiple *.geo files + # for a single day, the single scenes are renamed and placed in the GEO directory + #[yyyymmdd]_merged.orbtiming: *.orbtiming files for each day, to be used to find baselines between acquisitions + # merged_list: list of .geo files that have already been merged to ../GEO/*.geo (to avoid reprocessing) + # processed: list of *.zip files that have been processed + # preciseorbitfiles: list of available preciseorbitfiles for processing + + + # RAW/*.geo: original .geo files for each .zip file. Recommended to delete after processing to + # save space + # RAW/*.orbtiming*: orbtiming files for each .geo file. Recommended to delete after processing to save space + # RAW/*.SAFE: SAFE directory, unzipped *.zip file. Recommended to delete after processing to save space + + +import os +import subprocess +import sys +import time + +if len(sys.argv)<3: + print('Usage: generate_geo.py home path_to_merged REMOVE_RAW_GEO=N>') + sys.exit(0) + +##### ----- REQUIRED PARAMETERS ----- ##### +home = sys.argv[1] +path_to_merged = sys.argv[2] + +##### ----- DEFAULT PARAMETERS ----- ##### +update_flag = 'N' +REMOVE_ZIP = 'N' +REMOVE_SAFE = 'N' +REMOVE_ORBTIMING = 'N' +REMOVE_RAW_GEO = 'N' + +##### ----- CHECK FOR INPUTS ----- ##### + +if len(sys.argv)>3: + update_flag = sys.argv[3] + +if len(sys.argv)>4: + REMOVE_ZIP = sys.argv[4] + +if len(sys.argv)>5: + REMOVE_SAFE = sys.argv[5] + +if len(sys.argv)>6: + REMOVE_ORBTIMING = sys.argv[6] + +if len(sys.argv)>7: + REMOVE_RAW_GEO = sys.argv[7] + +print('\n You are reading creating *.geo files in: '+path_to_merged+'; and update_flag = '+update_flag) +print(' Your Home directory is: '+home) +print(' You will be processing with the following flags:') +print(' REMOVE_ZIP: '+REMOVE_ZIP) +print(' REMOVE_SAFE: '+REMOVE_SAFE) +print(' REMOVE_ORBTIMING: '+REMOVE_ORBTIMING) +print(' REMOVE_RAW_GEO: '+REMOVE_RAW_GEO) + +##### ----- RUN SCRIPT ----- ##### +# 1. Determine if you will be using the CPU or GPU version +print('\nRun the Processor -- sentinel_cpu.py or sentinel_gpu.py') +ret = os.system('which nvidia-smi') +if ret==0: + q=subprocess.Popen("nvidia-smi", stdout=subprocess.PIPE, shell=True) + (qq,err) = q.communicate() + if len(qq.decode())==0: + ret=-1 + +if ret == 0: + command = '$PROC_HOME/kp_scripts/sentinel/sentinel_gpu.py '+update_flag +else: + command = '$PROC_HOME/kp_scripts/sentinel/sentinel_cpu.py '+update_flag + + +# 2. Run through subdirectories and process L0 files, then Merge +ret = os.system('cp '+home+'/params .') +ret = os.system('cp '+home+'/.credentials .') +print('\nRunning Sentinel processor') +print('\n '+command) +ret = os.system(command) + +directory = os.getcwd() +print('\nIndividual SLCs generated in '+directory.strip()) + +# if needed, merge the SLCs +print('\n##### ----- Merging SLCs ----- #####') +if update_flag == 'Y': + command2 = 'rm '+path_to_merged+'/*.geo' + ret=os.system(command2) +command2 = '$PROC_HOME/kp_scripts/util/merge_slcs.py '+path_to_merged+' '+update_flag +print('\n '+command2) +ret = os.system(command2) + +# Delete completed files from the RAW directory to clear up space +print('\n##### ----- Cleaning Processing Directory ----- #####') +if REMOVE_ZIP == 'Y': + command2 = 'rm *.zip' + print(' '+command2) + ret=os.system(command2) + +if REMOVE_SAFE == 'Y': + command2 = 'rm -r *.SAFE' + print(' '+command2) + ret=os.system(command2) + +if REMOVE_ORBTIMING == 'Y': + command2 = 'rm *.orbtiming*' + print(' '+command2) + ret=os.system(command2) + +if REMOVE_RAW_GEO == 'Y': + command2 = 'rm *.geo' + print(' '+command2) + ret=os.system(command2) + +command2 = 'rm zipfiles ziplist* listofziplists' +print(' '+command2) +ret=os.system(command2) + + + + diff --git a/kp_scripts/int/int_coverage_mask.py b/kp_scripts/int/int_coverage_mask.py new file mode 100644 index 0000000..6312d57 --- /dev/null +++ b/kp_scripts/int/int_coverage_mask.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 + +# int_coverage_mask.py -- create a coverage (valid pixel) mask for each interferogram in an input intlist + +import sys +from datetime import datetime +import os +import math + +if len(sys.argv) < 2: + print ('Usage: int_coverage_mask.py intlist length') + sys.exit(1) + +intlist=sys.argv[1] +length=sys.argv[2] + +# create coverage mask for the files in the intlist +fint=open(intlist,'r') +ints=fint.readlines() +fint.close() + +for intfile in ints: + outfile = intfile.strip().replace('.int','.mask') + if os.path.isfile(outfile)==False: + # Make the mask for the interferogram + words=intfile.strip().split('_') + date1=words[0] + date2=words[1][:8] + mask1=date1+'_merged_multi.mask' + mask2=date2+'_merged_multi.mask' + command='$PROC_HOME/kp_scripts/int/int_simmask '+mask1+' '+mask2+' '+length+' '+outfile + print(' '+command) + ret=os.system(command) +print('\n Coverage Masks Generated For All Interferograms in List!') diff --git a/kp_scripts/int/int_simmask b/kp_scripts/int/int_simmask new file mode 100644 index 0000000..e8b1944 Binary files /dev/null and b/kp_scripts/int/int_simmask differ diff --git a/kp_scripts/int/int_simmask.f90 b/kp_scripts/int/int_simmask.f90 new file mode 100644 index 0000000..6cc8091 --- /dev/null +++ b/kp_scripts/int/int_simmask.f90 @@ -0,0 +1,82 @@ +! given two input mask filenames, generate a new mask file for interpolation +! (phase reconstruction) +! saves as output filename (which will most-often be temp_mask) + +! compile as: gfortran -o int_simmask int_simmask.f90 + +PROGRAM int_simmask + IMPLICIT none + + !specifications + INTEGER::i,j,stat,fstat,ierr,k + INTEGER*8 ::nr, naz !image size + INTEGER,DIMENSION(13)::statb + CHARACTER(200)::filename1,filename2,outfile,str + INTEGER*1,DIMENSION(:,:),ALLOCATABLE::mask1,mask2,newmask + !integer*1,dimension(:,:),allocatable :: newmask + + !executions + + !flist: list of wrapped intrerferogram files + !nfiles: number of files in flist + !nslcs: number of slc files + !len: width of the files + IF(iargc().lt.3)THEN + WRITE(*,*)'usage: int_simmask mask1 mask2 len ' + STOP + END IF + + CALL getarg(1,filename1) !mask name #1 + CALL getarg(2,filename2) !mask name #2 + CALL getarg(3,str) + READ(str,*)nr !width of files + outfile='temp_mask' + if(iargc().ge.4)then + call getarg(4,outfile) !new mask output filename + end if + + + ! Figure out how many lines there are in file + !open(unit=2, file=filename1, form="unformatted", access="direct", recl=nr*8) + open(unit=2,file=filename1,status='old',access='stream') + ierr=fstat(2,statb) + naz=statb(8)/nr + print*,naz + close(2) + + ALLOCATE(mask1(nr,naz),mask2(nr,naz)) + + ! Read mask 1 from file + open(unit=10,file=filename1,access='stream') + read(10)mask1 + close(10) + + ! Read mask 2 from file + open(unit=20,file=filename2,access='stream') + read(20)mask2 + close(20) + + ALLOCATE(newmask(nr,naz)) + + newmask=0 + k=0 + ! Create the new matrix (intersection of original two) using bitwise AND + do j=1,naz + do i=1,nr + if(mask1(i,j).ne.0.AND.mask2(i,j).ne.0)then + newmask(i,j) = -1 + k=k+1 + end if + end do + end do + + + ! Save new mask to file + open(unit=30,file=outfile, form="unformatted",status="replace",access="direct",recl=nr*naz) + write(30,rec=1)newmask + close(30) + + print *,'Saved interferogram mask to ',outfile + print *,'number of points ',k + +end program int_simmask diff --git a/kp_scripts/int/refpointsfromsim b/kp_scripts/int/refpointsfromsim new file mode 100644 index 0000000..e4bcef8 Binary files /dev/null and b/kp_scripts/int/refpointsfromsim differ diff --git a/kp_scripts/int/refpointsfromsim.f90 b/kp_scripts/int/refpointsfromsim.f90 new file mode 100644 index 0000000..1cde8fd --- /dev/null +++ b/kp_scripts/int/refpointsfromsim.f90 @@ -0,0 +1,69 @@ +! +! +! refpointsfromsim - find candidate reference points from existing similarity calculation +! +! note: needed amplitude info is in file 'scr' + +! Compile as: gfortran -o refpointsfromsim refpointsfromsim.f9 + +PROGRAM findrefpointssim + +implicit none +character*300 simfile, str, scrfile, out_reflocs, out_locs +real, allocatable :: locs(:,:), scr(:,:), similarity(:,:) +integer*1, allocatable :: bytemask(:,:) +integer nx, ny, i, k, kk, ii, jj, nrefs, ierr, j +real thresh + +if (iargc().lt.3)then + print *,'Usage: refpointsfromsim simfile scrfile nx ny out_reflocs out_locs ' + stop +end if + +call getarg(1,simfile) +call getarg(2,scrfile) +call getarg(3,str) +read(str,*)nx +call getarg(4,str) +read(str,*)ny +call getarg(5,out_reflocs) +call getarg(6,out_locs) + +thresh=0.6 +if (iargc().ge.7)then + call getarg(7,str) + read(str,*)thresh +end if + +ALLOCATE(scr(nx*2,ny),bytemask(nx,ny),similarity(nx,ny)) + + ! get similarity + open(15,file=simfile,access='stream') + read(15)similarity + close(15) + open(15,file=scrfile,access='stream') + read(15)scr + close(15) + + allocate (locs(nx,ny)) + open(20,file=out_reflocs) + open(21,file=out_locs,access='stream') + bytemask=0 + locs=0 + k=0 + do j=1,ny + do i=1,nx + if(similarity(i,j).ge.thresh)then + bytemask(i,j)=-1 + locs(i,j)=1. + k=k+1 + write(20,*)i,j + end if + end do + write(21)scr(1:nx,j),locs(:,j) + end do + close(20) + close(21) + print *,'Ref points found: ',k + +end PROGRAM findrefpointssim diff --git a/kp_scripts/int/tropocorrect.py b/kp_scripts/int/tropocorrect.py new file mode 100644 index 0000000..d686d13 --- /dev/null +++ b/kp_scripts/int/tropocorrect.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +# +# +# tropocorrect.py -- compensate troposphere using regression vs elevation +# +# INPUTS: +# unwlist: list of unwrapped files to be processed. Note: if they exist in the current +# directory, they will be overwritten. So, it's suggested that if you want to save +# both versions of the unwrapped interferograms for comparison, you will create a +# subdirectory, move to that directory, and have your unwrapped files in unwlist +# point to the files in the original directory. +# unwwidth: Width of unwrapped files +# unwlength: Length of unwrapped files +# +# + +import os +import sys + +if len(sys.argv)<4: + print ('Usage: tropocorrect.py unwlist unwwidth unwlength ') + sys.exit(0) + + +unwlist = sys.argv[1] +unwwidth = sys.argv[2] +unwlength = sys.argv[3] +thresh = '0.5' +if len(sys.argv) > 4: + thresh = sys.argv[4] + + +# regress vs hgt to remove tropo + +isave=0 +if isave==1: + command = 'mkdir unwrapped_orig_files' # save unwrapped files without correction + print (command) + ret = os.system(command) + command = 'cp *.unw unwrapped_orig_files/' + print (command) + ret = os.system(command) + +# do the regression +command = '$PROC_HOME/sentinel/regress_igrams.py unwlist dem '+unwwidth+' '+unwlength+' ref_locs' +print (command) +ret = os.system(command) + +print('Troposphere regression complete') diff --git a/kp_scripts/ml_scene_level.py b/kp_scripts/ml_scene_level.py new file mode 100644 index 0000000..4e6b65c --- /dev/null +++ b/kp_scripts/ml_scene_level.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 +# +# +# ml_scene_level.py -- Create Scene-level files for a multilooked study area +# +# +# INPUTS: + # geolist_file: list of full_resolution *.geo files + # looksac: Number of looks along x direction + # looksdn: Number of looks along y direction + # update_flag: if files need to be recreated, set update_flag = 'Y'. If files do NOT + # need to be recreated, keep update_flag = 'N'. (default = 'N') + # WATER_MASK: if you'd like to create a water mask and intersect it with the scene-level masks + # set to 'Y' (default = N). Note: isn't currently functional + # LOOK_VECTOR: if you'd like to create look vectors for each multilooked .geo file, set to 'Y' + # (default = 'N') + +# FILE REQUIREMENTS: + # params: parameters file in the current directory + +# GENERATES: + # *_multi.geo: multilooked geofiles with _multi tag + # geolist_ml: geolist equivalen to geolist_file, but pointing to the multilooked files + # dem: multilooked DEM in current directory + # dem.rsc: multilooked DEM rsc file in current directory + # *_multi.mask: a mask designating where the acquisition has valid coverage. Note: if WATER_MASK + # is 'Y' (and functional), then this mask includes valid coverage and land intersection + # mask_water: water maks designating where pixels are land (not currently working!) + # *.labels: Label file for look vector + # *.lookvector: Look vector file + +import os +import subprocess +import sys +import time + +if len(sys.argv)<4: + print('Usage: ml_scene_level.py geolist_file looksac looksdn ') + sys.exit(0) + +##### ----- REQUIRED PARAMETERS ----- ##### +geolist_file = sys.argv[1] +looksac = str(sys.argv[2]) +looksdn = str(sys.argv[3]) + +##### ----- DEFAULT PARAMETERS ----- ##### +update_flag = 'N' +WATER_MASK = 'N' +LOOK_VECTOR = 'N' + + +##### ----- CHECK FOR INPUTS ----- ##### + +if len(sys.argv)>4: + update_flag = sys.argv[4] + +if len(sys.argv)>5: + WATER_MASK = sys.argv[5] + +if len(sys.argv)>6: + LOOK_VECTOR = sys.argv[6] + +directory = os.getcwd() +print('\n You are reading creating multilooked, scene-level files in: '+directory+'; and update_flag = '+update_flag) +print(' Your number of looks: '+looksac+' across and '+looksdn+' down') +print(' You will be processing with the following flags:') +print(' WATER_MASK: '+WATER_MASK) +print(' LOOK_VECTOR: '+LOOK_VECTOR) + +##### ----- RUN SCRIPT ----- ##### +# 1. Determine the full-resolution DEM size +with open('params','r') as fparams: + demin = fparams.readline().strip() + rscin = fparams.readline().strip() + +with open(rscin,'r') as frsc: + words=frsc.readline() + demwidth=words.split()[1].strip() + words=frsc.readline() + demlength=words.split()[1].strip() + +# 2. Create Multilooked .geo files +print('\n ##### ----- Multilooking Geocoded SLCS ----- #####') +command = '$PROC_HOME/kp_scripts/util/multilook_slcs.py '+geolist_file+' '+demwidth+' '+looksac+' '+looksdn+' '+update_flag +print(' '+command) +ret=os.system(command) + +# 3. Create a new DEM and DEM.RSC file for Multilooked DEM/study area +if (os.path.isfile('dem.rsc')==False) or (update_flag=='Y'): + print('\n ##### ----- Multilooking DEM ----- #####') + command = '$PROC_HOME/kp_scripts/util/make_ml_demrsc.py '+rscin+' 1 1 '+demwidth+' '+demlength+' '+looksac+' '+looksdn + print(' '+command) + ret=os.system(command) + + # Create a reduced size DEM to match the multilooked files + command = '$PROC_HOME/util/nbymi2 '+demin+' dem '+demwidth+' '+looksac+' '+looksdn + print(' '+command) + ret=os.system(command) + +# 4. Get size of multilooked files from new rsc file +with open('dem.rsc','r') as fe: + words=fe.readline() + unwwidth=words.split()[1].strip() + words=fe.readline() + unwlength=words.split()[1].strip() + +# 5. Create geolist for the multilooked .geo files in the current directory +command = 'ls *.geo |cat> geolist_ml' +ret=os.system(command) + +# 6. Create Study area masks +print('\n ##### ----- Generating Coverage Masks for Each GEocoded SLC ----- #####') +command = '$PROC_HOME/kp_scripts/util/make_coverage_masks.py geolist_ml '+unwwidth+' '+unwlength+' '+update_flag +print(' '+command) +ret=os.system(command) + +# Make a water mask +if WATER_MASK=='Y': + #print('\n Generating Water Mask') + + # Merge the SLC coverage masks with the water mask (if designated) + #print(' Finding Intersection of Each SLC mask with Water Mask') + print('WATER MASK NOT CURRENTLY FUNCTIONL. TO BE UPDATED!') + +# 7. Generate Look Vectors if LOOK_VECTOR = Y +if LOOK_VECTOR == 'Y': + # Find the LOS look vector for each pixel, each multilooked .geo file + print('\n ##### ----- Calculating Look Vectors for Each Multilooked Geocoded SLC ----- #####') + + with open(geolist_file,'r') as fgeo: + geolist = fgeo.readlines() + for k in range(len(geolist)): + geolist[k] = geolist[k].strip() + + with open('orbitfiles','w') as forb: + orbits = [] + for k in range(len(geolist)): + newout = geolist[k].replace('.geo','.orbtiming') + forb.write(newout+'\n') + orbits.append(newout) + + norbs = len(orbits) + for k in range(norbs): + orbitname_in = orbits[k] + lv_out = orbitname_in.split('/')[-1].replace('orbtiming','lookvector') + + if (os.path.isfile(lv_out) == False) or (update_flag=='Y'): + command = '$PROC_HOME/sentinel/lookvector '+orbitname_in + print(' '+command) + ret=os.system(command) + command = 'mv '+orbitname_in.replace('orbtiming','labels')+' .' + ret=os.system(command) + command = 'mv '+orbitname_in.replace('orbtiming','lookvector')+' .' + ret=os.system(command) + else: + print(' '+lv_out+' already exists.') + print('\nLook Vectors Generated!') + + + + + + diff --git a/kp_scripts/original_sbas.py b/kp_scripts/original_sbas.py new file mode 100644 index 0000000..d94232e --- /dev/null +++ b/kp_scripts/original_sbas.py @@ -0,0 +1,200 @@ +#!/usr/bin/env python3 +# +# +# original_sbas.py -- Given a series of .geo files, generate SBAS time series with basic workflow +# - Each interferogram is treated identically +# - All pixels are used in unwrapping +# - Cosine_similarity index is used to find reference pixels for tropospheric correction & calibration +# - SBAS includes capabilities for consistent and inconsistent coverage +# +# +# INPUTS: + +# FILE REQUIREMENTS: + +# GENERATES: + + +import os +import subprocess +import sys +import time + +##### ----- CHECK USAGE ----- ##### + +geolist_file = sys.argv[1] +path_to_GEO = sys.argv[2] +max_tb = sys.argv[3] +max_sb = sys.argv[4] +looksac = sys.argv[5] +looksdn = sys.argv[6] +scenemask = sys.argv[7] +INCONSISTENT = sys.argv[8] +psthresh = sys.argv[9] +simthresh = sys.argv[10] +refthresh = sys.argv[11] +TROPO_CORR = sys.argv[12] + +##### ----- DEFAULT PARAMETERS ----- ##### + + +##### ----- CHECK FOR INPUTS ----- ##### + +print('\n You are reading geolist: '+geolist_file) +print(' You will be processing with the following parameters:') +print(' Max Temporal Baseline: '+max_tb) +print(' Max Spatial Baseline: '+max_sb) +print(' scenemask: '+scenemask) + +if INCONSISTENT == 'N': + print(' Coverage Flag: Consistent') + flag = '0' +else: + print(' Coverage Flag: Inconsistent') + flag = '1' +print(' Tropospheric Correction: '+TROPO_CORR) + +##### ----- RUN SCRIPT ----- ##### +# 1. Make SBAS list given max_tb, max_sb, and geolist +print('\n Making sbaslist and intlist') +command = '$PROC_HOME/kp_scripts/sentinel/sbas_list.py '+geolist_file+'_fullres '+path_to_GEO+' '+'sbaslist_'+max_tb+' '+max_tb+' '+max_sb +print(' '+command) +ret=os.system(command) + +command = '$PROC_HOME/kp_scripts/sbas/intlist_from_sbas.py sbaslist_'+max_tb +print(' '+command) +ret=os.system(command) + +# 2. Read in the params file and elevation.dem.rsc file +with open('params','r') as fpar: + dem = fpar.readline().strip() + rsc = fpar.readline().strip() + +with open(rsc,'r') as frsc: + demwidth = frsc.readline().strip().split()[-1] + demlength = frsc.readline().strip().split()[-1] + +with open('dem.rsc','r') as fe: + unwwidth = fe.readline().strip().split()[-1] + unwlength = fe.readline().strip().split()[-1] + +# 3. Make the interferograms +print('\n Making interferograms in sbaslist_'+max_tb) +command = '$PROC_HOME/kp_scripts/sentinel/ps_sbas_igrams.py sbaslist_'+max_tb+' '+rsc+' 1 1 '+demwidth+' '+demlength+' '+looksac+' '+looksdn +print(' '+command) +ret=os.system(command) + +# 4. Make All Similarity Mask +suff = '_'+max_tb +print('\n Making all_similarity_mask for intlist_'+max_tb+' and '+geolist_file) +command = '$PROC_HOME/kp_scripts/ps/cosine_sim_mask intlist_'+max_tb+' '+unwwidth+' pssim'+suff+' '+scenemask+' '+flag+' '+psthresh+' '+simthresh+' '+suff +print(' '+command) +ret=os.system(command) + +# 5. Make ref_locs file +print('\n Finding reference pixels from cosine similarity file') +simfile = 'all_similarity'+suff +scrfile = 'scr'+suff +out_reflocs = 'ref_locs'+suff +out_locs = 'locs'+suff + +command = '$PROC_HOME/kp_scripts/int/refpointsfromsim '+simfile+' '+scrfile+' '+unwwidth+' '+unwlength+' '+out_reflocs+' '+out_locs+' '+refthresh +print(' '+command) +ret=os.system(command) + +# 5. Unwrap Interferograms +print('\n ##### ----- UNWRAPPING INTERFEROGRAMS IN PARALLEL ----- #####') +command = '$PROC_HOME/kp_scripts/util/unwrap_parallel.py intlist_'+max_tb+' '+unwwidth+' . '+INCONSISTENT+' '+scenemask +print(' '+command) +ret=os.system(command) + +# Make an unwlist file +funw = open('unwlist_'+max_tb,'w') +nints = 0 +unwfiles = [] +with open('intlist_'+max_tb,'r') as fint: + for line in fint.readlines(): + outname = line.replace('.int','.unw') + funw.write(outname) + unwfiles.append(outname.strip()) + nints+=1 +funw.close() +nints=str(nints) + +# 6. Tropospheric Correction +if TROPO_CORR == 'Y': + print('\n ##### ----- APPLYING TROPOSPHERIC CORRECTION TO INTERFEROGRAMS ----- #####') + if os.path.isdir('tropo_corr') == False: + ret=os.system('mkdir tropo_corr') + + # Change to directory where you want the files saved + os.chdir('tropo_corr') + + # Copy intlist to current directory and make corresponding unwlist + ret=os.system('cp ../intlist_'+max_tb+' .') + with open('intlist_'+max_tb,'r') as fint: + ints = fint.readlines() + + with open('unwlist_'+max_tb,'w') as funw: + for name in ints: + unwfile = '../'+name.replace('.int','.unw') + funw.write(unwfile) + + # Run the regression + command = '$PROC_HOME/kp_scripts/util/regress_igrams.py unwlist_'+max_tb+' ../dem '+unwwidth+' '+unwlength+' ../ref_locs_'+max_tb+' '+INCONSISTENT + print(' '+command) + ret=os.system(command) + + ret=os.chdir('..') + +# 7. SBAS Setup +print('\n ##### ----- MAKING SBAS SETUP FILES ----- #####') +command = '$PROC_HOME/kp_scripts/sbas/sbas_setup.py sbaslist_'+max_tb+' '+geolist_file+' _'+max_tb +print(' '+command) +ret=os.system(command) + +# Find number of slcs +with open(geolist_file,'r') as fgeo: + nslc=str(len(fgeo.readlines())) + + + +# 8. Run SBAS +print('\n ##### ----- RUNNING SBAS ON ORIGINAL INTERFEROGRAMS ----- #####') +suffix = '_'+max_tb +unwlistin = 'unwlist'+suffix +Tmfile = 'Tm'+suffix+'.out' +timedeltafile = 'timedeltas'+suffix+'.out' +deltimefile = 'deltime'+suffix+'.out' +reflocs_file = 'ref_locs'+suffix +if TROPO_CORR == 'Y': + unwlistin = '../'+unwlistin + Tmfile = '../'+Tmfile + timedeltafile = '../'+timedeltafile + deltimefile = '../'+deltimefile + reflocs_file = '../'+reflocs_file + scenemask = '../'+scenemask + ret=os.chdir('tropo_corr') + + for unw in unwfiles: + command = 'mv ../'+unw.replace('.unw','.amp')+' .' + ret = os.system(command) + if INCONSISTENT=='Y': + command = 'mv ../'+unw.replace('.unw','.mask')+' .' + ret=os.system(command) + + +command = '$PROC_HOME/kp_scripts/sbas/sbas_unique '+unwlistin+' '+nints+' '+nslc+' '+unwwidth+' '+Tmfile+' '+timedeltafile+' '+deltimefile+' '+reflocs_file+' '+scenemask+' '+INCONSISTENT +print(' '+command) +ret=os.system(command) + +if TROPO_CORR == 'Y': + os.system('mv *.amp *.mask ..') + os.chdir('..') + +print('\n !!!!! ----- ORIGINAL SBAS WORKFLOW COMPLETE ----- !!!!!') + +sys.exit(0) + + + diff --git a/kp_scripts/ps/cosine_sim_mask b/kp_scripts/ps/cosine_sim_mask new file mode 100644 index 0000000..7d352d1 Binary files /dev/null and b/kp_scripts/ps/cosine_sim_mask differ diff --git a/kp_scripts/ps/cosine_sim_mask.f90 b/kp_scripts/ps/cosine_sim_mask.f90 new file mode 100644 index 0000000..07cfe61 --- /dev/null +++ b/kp_scripts/ps/cosine_sim_mask.f90 @@ -0,0 +1,1255 @@ +! run ps detection using cosine similarlity on a set of wrapped files +! Read also correlation and amp files +! Read also mask files (if designated), in order to account for inconsistent +! coverage +! this version allows for averaging multiple reference points if file is supplied +! modified to use fftw3 28jan23 + +! Compile as: gfortran -o cosine_sim_mask cosine_sim_mask.f90 -fopenmp +! $PROC_HOME/fft/fftw-3.3.9/.libs/libfftw3f.a + +PROGRAM cosine_sim_mask + use omp_lib + + IMPLICIT none + + !!!!! ----- Specifications: Declare your variables ----- !!!!! + INTEGER::i,j,r,c,rows,cols,stat,fstat,ierr,looks,k,kk,n_refs,igramnumber,iter,maskflag + real*8 jdprimary, jdsecondary + INTEGER*8 ::nr, naz, reclphase, recsize, planf, plani !image size + INTEGER::ncells !number of cells (files in flist) + INTEGER,DIMENSION(13)::statb + CHARACTER(200)::filelist,str,ref_locs_file, outfile, scenemaskfile, suffix + CHARACTER(200),DIMENSION(:),ALLOCATABLE::cells !array of file names + CHARACTER(200)::strint,strunw,stramp,strcc,strmask + REAL,DIMENSION(:,:,:),ALLOCATABLE::phase,amps !,coh,temp3 + REAL,DIMENSION(:,:),ALLOCATABLE::amp,dat,mask,temp2,stack,stacktime,scr,similarity,allsimilarity + complex*8,dimension(:,:,:),allocatable :: igrams + real*4 psthresh,simthresh,similaritymean + integer*1,dimension(:,:),allocatable :: bytemask,tempmask,scenemask + integer*1,dimension(:,:,:),allocatable :: masks + + !!!!! ----- Executions ----- !!!!! + + ! filelist: list of wrapped intrerferogram files, typically ending in *.int + ! len: width of the files + ! outfile: name of output all_similarity_mask + ! scenemaskfile: name of input scenemask + ! maskflag: Determines if individual interferogram files are loaded in (inconsistent SLC coverage) + ! 0=NO individual interferogram files needed + ! 1=individual interferograms masks are loaded in + ! : optional, threshold value for designating PS pixels from MLEstimator + ! : optional, threshold cosine similarity value for designating PS pixels from cosine similarity index + ! : optional, tag string onto end of output files, to be more specific + + IF(iargc().lt.3)THEN + WRITE(*,*)'usage: cosine_sim_mask igramlist len outfile scenemaskfile maskflag ' + STOP + END IF + + CALL getarg(1,filelist) + CALL getarg(2,str) + READ(str,*)nr !width of files + call getarg(3,outfile) + call getarg(4,scenemaskfile) + call getarg(5,str) + READ(str,*)maskflag + + psthresh=2. + simthresh=0.5 + suffix = '' + if(iargc().ge.6)then + call getarg(6,str) + read(str,*)psthresh + end if + if(iargc().ge.7)then + call getarg(7,str) + read(str,*)simthresh + end if + if(iargc().ge.8)then + call getarg(8,suffix) + end if + + !!!!! ----- Determine the number of interferograms in igramlist ----- !!!!! + ALLOCATE(cells(10000)) + + OPEN(UNIT=1,FILE=filelist,STATUS='old') + do ncells=1,10000 + READ(1,'(A)',end=10,IOSTAT=stat)cells(ncells) + end do +10 continue + CLOSE(1) + ncells=ncells-1 + + !!!!! ----- Determine the height of the ints (# of azimuth lines) ----- !!!!! + OPEN(UNIT=2,FILE=cells(1),FORM='unformatted',ACCESS='direct',RECL=nr*8) + ierr=fstat(2,statb) + naz=statb(8)/8/nr + CLOSE(2) + print *,'Number of interferograms: ',ncells,', Lines per file: ',naz + + !!!!! ----- Allocate important arrays ----- !!!!! + ALLOCATE(igrams(nr,naz,ncells),masks(nr,naz,ncells),scr(nr,naz),similarity(nr,naz),allsimilarity(nr,naz)) + ALLOCATE(amp(nr,naz),bytemask(nr,naz),amps(nr,naz,ncells)) + + !read in wrapped igrams + !$omp parallel do private(strint,strmask,tempmask) shared(igrams,ncells,amps,masks) + DO i=1,ncells + + ! read in the interferogram + strint=trim(adjustl(cells(i))) + strmask=Replace_Text(strint,'.int','.mask') + OPEN(UNIT=i+10,FILE=strint,FORM='unformatted',ACCESS='direct',RECL=2*nr*naz*4,CONVERT='LITTLE_ENDIAN') + READ(i+10,rec=1)igrams(:,:,i) + CLOSE(i+10) + amps(:,:,i)=cabs(igrams(:,:,i))**2 + + ! If maskflag == 1 then load in each individual interferogram mask. The + ! name is the interferogram filename with .int replaced by .mask + if (maskflag==1)then + + allocate(tempmask(nr,naz)) + + ! read the corresponding byte mask file + open(unit=i+10,file=strmask,access='stream') + read(i+10)tempmask + CLOSE(i+10) + masks(:,:,i)=tempmask + + ! free up some space + deallocate(tempmask) + end if + + END DO + !$omp end parallel do + amp=sqrt(sum(amps,3)/ncells) ! save amplitude average for later use if desred + deallocate(amps) + if(maskflag==0)then + DEALLOCATE(masks) + end if + + ! Load in the scenemask + allocate(scenemask(nr,naz)) + print*, 'loading scene mask: ',scenemaskfile + ! read the mask + open(unit=11,file=scenemaskfile,access='stream') + read(11)scenemask + CLOSE(11) + + !!!!! ----- STEP 1: MLE for Candidate PS Pixels ----- !!!!! + ! call mlesub to + ! 1. Filter the interferograms in igrams + ! 2. Compute signal-to-clutter-ratio for valid points + ! 3. Free up some memory + r=nr + c=naz + if(maskflag==1)then + call mlesub_intmask(igrams,masks,scenemask,r,c,ncells,scr) + else + call mlesub(igrams,scenemask,r,c,ncells,scr) + end if + + allocate(phase(nr,naz,ncells)) + !$omp parallel do shared(phase,igrams,ncells) + do i=1,ncells + phase(:,:,i)=atan2(aimag(igrams(:,:,i)),real(igrams(:,:,i))) + end do + !$omp end parallel do + deallocate (igrams) + +!Save nr, naz, nslc, ncells parameters in one file + OPEN(15,FILE='cosine_sim_parameters' // suffix,STATUS='replace') + WRITE(15,*) nr, naz, ncells, psthresh, simthresh + CLOSE(15) + + PRINT*,'Mle pass complete' + ! save scr for later analysis + open(15,file='scr_floats' // suffix,access='stream') + write(15)scr + close(15) + + open(15,file='scr' // suffix,access='stream') + do i=1,naz + write(15)amp(:,i),scr(:,i) + end do + close(15) + print *,'Saved scr estimates as floats (scr_floats) and mht (scr) formats' + + ! save byte mask for scr's above threshold + bytemask=0 + k=0 + do j=1,naz + do i=1,nr + if(scr(i,j).ge.psthresh)then + bytemask(i,j)=-1 + k=k+1 + end if + end do + end do + open(15,file='scr_mask' // suffix,access='stream') + write(15)bytemask + close(15) + print *,'Saved scr mask as scr_mask, number= ',k + + + !!!!! ----- Step 2. Remove False Positives ----- !!!!! + ! Use cosine similarity filter to remove false positive PS pixels + ! compute median_similarity - for each ps pixel (>th), compute its phase similarity with + ! nearby PS and return the median of similarity measurements + ! note: phase has been calculated using the filtered interferograms (during + ! mlesub) + if(maskflag==1)then + call median_similarity_intmask(phase,masks,scenemask,scr,r,c,ncells,psthresh,similarity) + else + call median_similarity(phase,scenemask,scr,r,c,ncells,psthresh,similarity) + end if + + ! save median similarity for later analysis + open(15,file='median_similarity' // suffix,access='stream') + write(15)similarity + close(15) + print *,'Saved median similarity (median_similarity) as floats with ps threshold ',psthresh + + ! save byte mask for median similarity above threshold + bytemask=0 + k=0 + similaritymean=0. + do j=1,naz + do i=1,nr + if(similarity(i,j).ge.simthresh)then + bytemask(i,j)=-1 + k=k+1 + similaritymean=similaritymean+similarity(i,j) + end if + end do + end do + similaritymean=similaritymean/k + open(15,file='median_similarity_mask' // suffix,access='stream') + write(15)bytemask + close(15) + print *,'Saved median similarity mask as median_sim_mask, number, mean= ',k,similaritymean + + !!!!! ----- Step 3. Find Additional PS Pixels with All Similarity ----- !!!!! + ! scan all points for similarity with refined ps set, adding extra discoveries + ! and now get the similarity everywhere, adding in newly discovered points + + do iter=1,3 + print *,'Iteration ',iter + if(iter.gt.1)similarity=allsimilarity + if(maskflag==1)then + call all_similarity_intmask(phase,masks,scenemask,similarity,r,c,ncells,simthresh,allsimilarity) + else + call all_similarity(phase,scenemask,similarity,r,c,ncells,simthresh,allsimilarity) + end if + end do + + ! save in mht format + open(15,file=trim(outfile) // suffix,access='stream') + do i=1,naz + write(15)amp(:,i),allsimilarity(:,i) + end do + ! and also as floats + open(15,file='all_similarity' // suffix,access='stream') + write(15)allsimilarity + close(15) + print *,'Saved final similarity as floats (all_similarity) and mht ',trim(outfile),' with sim threshold ',simthresh + ! and finally a byte mask over the sim_thresh + bytemask=0 + k=0 + similaritymean=0. + kk=0 + do j=1,naz + do i=1,nr + if(allsimilarity(i,j).ge.simthresh)then + bytemask(i,j)=-1 + k=k+1 + similaritymean=similaritymean+similarity(i,j) + end if + if(abs(allsimilarity(i,j)).ge.1.e-3)then + kk=kk+1 + end if + end do + end do + similaritymean=similaritymean/k + open(15,file='all_similarity_mask' // suffix,access='stream') + write(15)bytemask + close(15) + print *,'Saved all_similarity mask as all_similarity_mask, number, mean= ',k,similaritymean + print *,'Fractional coverage: ',float(k)/float(naz)/float(nr) + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +!FUNCTIONS + +CONTAINS + + !strrep(str,orig,rep): replaces all instaces of orig in str with rep + CHARACTER(30) FUNCTION strrep(str,orig,rep) + IMPLICIT none + !specifications + INTEGER::i + CHARACTER(100),INTENT(IN)::str + CHARACTER(2),INTENT(IN)::orig,rep + !executions + strrep(:) = str(:) + DO i=1,LEN(strrep)-1 + IF(strrep(i:i+1) == orig(:))THEN + strrep(i:i+1) = rep(:) + END IF + END DO + END FUNCTION strrep + +! ------------------ +FUNCTION Replace_Text (s,text,rep) RESULT(outs) +CHARACTER(*) :: s,text,rep +CHARACTER(LEN(s)+100) :: outs ! provide outs with extra 100 char len +INTEGER :: i, nt, nr + +outs = s ; nt = LEN_TRIM(text) ; nr = LEN_TRIM(rep) +DO + i = INDEX(outs,text(:nt)) ; IF (i == 0) EXIT + outs = outs(:i-1) // rep(:nr) // outs(i+nt:) +END DO +END FUNCTION Replace_Text + + !mean2d(mat): calculates the mean value of a 2D matrix mat with r rows and c columns + REAL FUNCTION mean2d(mat,rows,cols) + IMPLICIT none + REAL,INTENT(IN),DIMENSION(:,:) :: mat + INTEGER :: r,c,rows,cols + mean2d=0 + DO r=1,rows + DO c=1,cols + mean2d=mean2d+mat(r,c) + END DO + END DO + mean2d=mean2d/SIZE(mat) + END FUNCTION mean2d + + subroutine mlesub_intmask(igrams,masks,scenemask,len,lines,nigrams,scrout) +!c mlestack - apply mle estimator to phases from stacked time series of igrams + + use omp_lib + implicit none + complex*8, allocatable :: in(:,:),igram(:,:),igramfilt(:,:),filt(:,:),work(:,:) + complex*8 csum,igrams(len,lines,nigrams) + integer*1 masks(len,lines,nigrams),scenemask(len,lines) + integer*1, allocatable :: inmask(:,:) + complex*8, allocatable :: valid(:) + real*4, allocatable :: out(:),phase(:) + real*4 scrout(len,lines) + real*4 pdfs(200,100),prob(100) + real*4 pi,rho,rhoest,thresh,probmax,pow,ph,screst,x,y,dx,aveps,avemag,box,aveph,amp,scr + integer*4, allocatable :: iph(:) + character*100 flist,fin(10000),fout,str + integer statb(13),fstat,len,lines,nigrams,ibox,nvalid,kmax,krho,ixfft,iyfft,iy,ix,line,ixbins + + !$omp parallel + ! n=omp_get_num_threads() + !$omp end parallel + !print *, 'Max threads used: ', n + + ! default to box size 10 + box=10. + +! print *,'Interferograms analyzed: ',nigrams + +!c get sizes for ffts + do i=1,20 + if(len.gt.2**i)ixfft=2**i + if(lines.gt.2**i)iyfft=2**i + end do + ixfft=ixfft*2 + iyfft=iyfft*2 +! print *,'FFT sizes: ',ixfft,iyfft + +!c allocate memory + allocate(out(len*2)) + allocate(in(len,nigrams)) + allocate(inmask(len,nigrams)) + allocate(phase(nigrams)) + allocate(iph(nigrams)) + allocate(valid(nigrams)) + allocate(filt(ixfft,iyfft)) + +!c precompute slc pdfs as function of rho and theta + ixbins=200 + pi=4*atan2(1.,1.) + dx=2*pi/ixbins + + do k=1,100 + rho=(k-1)/100. + scr=1/(1./rho-1.) +! print *,rho,scr + do i=1,ixbins + ph=-pi+dx/2.+(i-1)*dx + pdfs(i,k)=1./2./pi*exp(-scr*(sin(ph)**2))*(exp(-scr*cos(ph)**2)+ & + sqrt(pi*scr)*cos(ph)*(1-erf(-sqrt(scr)*cos(ph)))) + end do + end do + +!c make a filter directly in frequency domain + do ix=1,ixfft + if(ix.le.ixfft/2)then + x=sinc((ix-1)/(ixfft/box)) + else + x=sinc((ixfft-ix+1)/(ixfft/box)) + end if + + do iy=1,iyfft + if(iy.le.iyfft/2)then + y=sinc((iy-1)/(iyfft/box)) + else + y=sinc((iyfft-iy+1)/(iyfft/box)) + end if + + filt(ix,iy)=cmplx(x*y, 0.) + + end do + end do + + filt=conjg(filt) + + !!!!! ----- Filter Interferograms ----- !!!!! + allocate(igramfilt(ixfft,iyfft)) + call sfftw_plan_dft_2d(planf,ixfft,iyfft,igramfilt,igramfilt,-1,64) + call sfftw_plan_dft_2d(plani,ixfft,iyfft,igramfilt,igramfilt,+1,64) + deallocate(igramfilt) + !$omp parallel do private(i,igram,igramfilt,kk,k) & + !$omp shared(nigrams,lines,len,filt,ixfft,iyfft,fin,igrams,planf,plani) + do i=1,nigrams + if(mod(i,100).eq.1)print *,'Filtering igram ',i + allocate(igramfilt(ixfft,iyfft)) + igramfilt=cmplx(0.,0.) + igramfilt(1:len,1:lines)=igrams(:,:,i) + call sfftw_execute_dft(planf,igramfilt,igramfilt) + igramfilt=igramfilt*filt + call sfftw_execute_dft(plani,igramfilt,igramfilt) + igramfilt(1:len,1:lines)=igramfilt(1:len,1:lines)/cabs(igramfilt(1:len,1:lines)) + do k=1,len + !$omp parallel do private(kk) shared(igramfilt) + do kk=1,lines + if(isnan(real(igramfilt(k,kk))).or.isnan(aimag(igramfilt(k,kk))))then + igramfilt(k,kk)=cmplx(0.,0.) + end if + end do + !$omp end parallel do + end do + igrams(1:len,1:lines,i)=igrams(1:len,1:lines,i)*conjg(igramfilt(1:len,1:lines))!/cabs(igramfilt(1:len,1:lines)) + deallocate (igramfilt) + end do + !$omp end parallel do + call sfftw_destroy_plan(planf) + call sfftw_destroy_plan(plani) + + !!!!! ----- Calculate SCR at Valid Pixels ----- !!!!! +!c loop over lines + !$omp parallel do private(line,in,inmask,csum,i,k,pow,amp,phase,aveph) & + !$omp private(iph,probmax,krho,prob,kmax,rhoest,screst,out) & + !$omp private(nvalid,valid,thresh,avemag) & + !$omp shared(lines,nigrams,len,pi,ixbins,pdfs,igrams,masks,scrout,scenemask) + do line=1,lines + if(mod(line,1000).eq.1)print *,'At line: ',line + + !c read line from each interferogram + do i=1,nigrams + in(:,i)=igrams(:,line,i) + inmask(:,i)=masks(:,line,i) + end do + + !c loop over pixels in the line + do k=1,len + ! if the scenemask at this pixel is not 0 then do the calculation + if (scenemask(k,line)==0)then + out(k)=0 + out(k+len)=0 + cycle + end if + !c make a list of valid points + avemag=0. + do i=1,nigrams + if(inmask(k,i).ne.0)then + avemag=avemag+cabs(in(k,i)) + end if + end do + thresh=avemag/1000. + nvalid=0 + do i=1,nigrams + if (cabs(in(k,i)).ge.thresh.and.inmask(k,i).ne.0)then + nvalid=nvalid+1 + valid(nvalid)=in(k,i) + end if + end do + + if (nvalid.gt.1) then + !c force each sequence to zero mean + csum=sum(valid(1:nvalid)) + csum=csum/cabs(csum) + valid(1:nvalid)=valid(1:nvalid)*conjg(csum) + + !c sqrt(average powers) to store as amplitude value + pow=0. + pow=sum(cabs(valid(1:nvalid))**2) + amp=sqrt(pow/nvalid) + if(isnan(amp))amp=0. + + !c get the phase series + phase(1:nvalid)=atan2(aimag(valid(1:nvalid)),real(valid(1:nvalid))) + + !c apply mle + aveph=sum(phase(1:nvalid)) + aveph=aveph/nvalid + do i=1,nvalid + iph(i)=(phase(i)+pi)/2./pi*ixbins+1 + if(iph(i).lt.-100000)iph(i)=1 + end do + probmax=-1.e12 + !!!$omp parallel do private(krho,prob,i) + !!!$omp+ shared(pdfs,iph,probmax,nvalid) + do krho=1,100 + prob(krho)=0. + do i=1,nvalid + prob(krho)=prob(krho)+alog(pdfs(iph(i),krho)+1.e-12) + end do + if(prob(krho).ge.probmax)then + kmax=krho + probmax=prob(krho) + end if + end do + !!!$omp end parallel do + rhoest=(kmax-1)/100. + screst=1./(1./rhoest-1.) + + !c save result for pixel + out(k)=amp + out(k+len)=screst + else + out(k)=0 + out(k+len)=0 + end if + + end do + + scrout(:,line)=out(len+1:len*2) + end do + !$omp end parallel do + return + end subroutine mlesub_intmask + + + + subroutine mlesub(igrams,scenemask,len,lines,nigrams,scrout) +!c mlestack - apply mle estimator to phases from stacked time series of igrams + + use omp_lib + implicit none + complex*8, allocatable ::in(:,:),igram(:,:),igramfilt(:,:),filt(:,:),work(:,:) + complex*8 csum,igrams(len,lines,nigrams) + integer*1 scenemask(len,lines) + integer*1, allocatable :: inmask(:,:) + complex*8, allocatable :: valid(:) + real*4, allocatable :: out(:),phase(:) + real*4 scrout(len,lines) + real*4 pdfs(200,100),prob(100) + real*4 pi,rho,rhoest,thresh,probmax,pow,ph,screst,x,y,dx,aveps,avemag,box,aveph,amp,scr + integer*4, allocatable :: iph(:) + character*100 flist,fin(10000),fout,str + integer statb(13),fstat,len,lines,nigrams,ibox,nvalid,kmax,krho,ixfft,iyfft,iy,ix,line,ixbins + + !$omp parallel + ! n=omp_get_num_threads() + !$omp end parallel + !print *, 'Max threads used: ', n + + ! default to box size 10 + box=10. + +! print *,'Interferograms analyzed: ',nigrams + +!c get sizes for ffts + do i=1,20 + if(len.gt.2**i)ixfft=2**i + if(lines.gt.2**i)iyfft=2**i + end do + ixfft=ixfft*2 + iyfft=iyfft*2 +! print *,'FFT sizes: ',ixfft,iyfft + +!c allocate memory + allocate(out(len*2)) + allocate(in(len,nigrams)) + allocate(inmask(len,nigrams)) + allocate(phase(nigrams)) + allocate(iph(nigrams)) + allocate(valid(nigrams)) + allocate(filt(ixfft,iyfft)) + +!c precompute slc pdfs as function of rho and theta + ixbins=200 + pi=4*atan2(1.,1.) + dx=2*pi/ixbins + + do k=1,100 + rho=(k-1)/100. + scr=1/(1./rho-1.) +! print *,rho,scr + do i=1,ixbins + ph=-pi+dx/2.+(i-1)*dx + pdfs(i,k)=1./2./pi*exp(-scr*(sin(ph)**2))*(exp(-scr*cos(ph)**2)+ & + sqrt(pi*scr)*cos(ph)*(1-erf(-sqrt(scr)*cos(ph)))) + end do + end do + +!c make a filter directly in frequency domain + do ix=1,ixfft + if(ix.le.ixfft/2)then + x=sinc((ix-1)/(ixfft/box)) + else + x=sinc((ixfft-ix+1)/(ixfft/box)) + end if + + do iy=1,iyfft + if(iy.le.iyfft/2)then + y=sinc((iy-1)/(iyfft/box)) + else + y=sinc((iyfft-iy+1)/(iyfft/box)) + end if + + filt(ix,iy)=cmplx(x*y, 0.) + + end do + end do + + filt=conjg(filt) + + !!!!! ----- Filter Interferograms ----- !!!!! + allocate(igramfilt(ixfft,iyfft)) + call sfftw_plan_dft_2d(planf,ixfft,iyfft,igramfilt,igramfilt,-1,64) + call sfftw_plan_dft_2d(plani,ixfft,iyfft,igramfilt,igramfilt,+1,64) + deallocate(igramfilt) + !$omp parallel do private(i,igram,igramfilt,kk,k) & + !$omp shared(nigrams,lines,len,filt,ixfft,iyfft,fin,igrams,planf,plani) + do i=1,nigrams + if(mod(i,100).eq.1)print *,'Filtering igram ',i + allocate(igramfilt(ixfft,iyfft)) + igramfilt=cmplx(0.,0.) + igramfilt(1:len,1:lines)=igrams(:,:,i) + call sfftw_execute_dft(planf,igramfilt,igramfilt) + igramfilt=igramfilt*filt + call sfftw_execute_dft(plani,igramfilt,igramfilt) + igramfilt(1:len,1:lines)=igramfilt(1:len,1:lines)/cabs(igramfilt(1:len,1:lines)) + do k=1,len + !$omp parallel do private(kk) shared(igramfilt) + do kk=1,lines + if(isnan(real(igramfilt(k,kk))).or.isnan(aimag(igramfilt(k,kk))))then + igramfilt(k,kk)=cmplx(0.,0.) + end if + end do + !$omp end parallel do + end do + igrams(1:len,1:lines,i)=igrams(1:len,1:lines,i)*conjg(igramfilt(1:len,1:lines))!/cabs(igramfilt(1:len,1:lines)) + deallocate (igramfilt) + end do + !$omp end parallel do + call sfftw_destroy_plan(planf) + call sfftw_destroy_plan(plani) + + !!!!! ----- Calculate SCR at Valid Pixels ----- !!!!! +!c loop over lines + !$omp parallel do private(line,in,inmask,csum,i,k,pow,amp,phase,aveph) & + !$omp private(iph,probmax,krho,prob,kmax,rhoest,screst,out) & + !$omp private(nvalid,valid,thresh,avemag) & + !$omp shared(lines,nigrams,len,pi,ixbins,pdfs,igrams,scrout,scenemask) + do line=1,lines + if(mod(line,1000).eq.1)print *,'At line: ',line + + !c read line from each interferogram + do i=1,nigrams + in(:,i)=igrams(:,line,i) + end do + + !c loop over pixels in the line + do k=1,len + ! if the scenemask at this pixel is not 0 then do the calculation + if (scenemask(k,line)==0)then + out(k)=0 + out(k+len)=0 + cycle + end if + !c make a list of valid points + avemag=0. + do i=1,nigrams + avemag=avemag+cabs(in(k,i)) + end do + thresh=avemag/1000. + nvalid=0 + do i=1,nigrams + if (cabs(in(k,i)).ge.thresh)then + nvalid=nvalid+1 + valid(nvalid)=in(k,i) + end if + end do + + if (nvalid.gt.1) then + !c force each sequence to zero mean + csum=sum(valid(1:nvalid)) + csum=csum/cabs(csum) + valid(1:nvalid)=valid(1:nvalid)*conjg(csum) + + !c sqrt(average powers) to store as amplitude value + pow=0. + pow=sum(cabs(valid(1:nvalid))**2) + amp=sqrt(pow/nvalid) + if(isnan(amp))amp=0. + + !c get the phase series + phase(1:nvalid)=atan2(aimag(valid(1:nvalid)),real(valid(1:nvalid))) + + !c apply mle + aveph=sum(phase(1:nvalid)) + aveph=aveph/nvalid + do i=1,nvalid + iph(i)=(phase(i)+pi)/2./pi*ixbins+1 + if(iph(i).lt.-100000)iph(i)=1 + end do + probmax=-1.e12 + !!!$omp parallel do private(krho,prob,i) + !!!$omp+ shared(pdfs,iph,probmax,nvalid) + do krho=1,100 + prob(krho)=0. + do i=1,nvalid + prob(krho)=prob(krho)+alog(pdfs(iph(i),krho)+1.e-12) + end do + if(prob(krho).ge.probmax)then + kmax=krho + probmax=prob(krho) + end if + end do + !!!$omp end parallel do + rhoest=(kmax-1)/100. + screst=1./(1./rhoest-1.) + + !c save result for pixel + out(k)=amp + out(k+len)=screst + else + out(k)=0 + out(k+len)=0 + end if + + end do + + scrout(:,line)=out(len+1:len*2) + end do + !$omp end parallel do + return + end subroutine mlesub + + real function sinc(q) + real*4 q,pi + parameter (pi=3.14159265359) + + if(abs(q).le.1.e-6)then + sinc=1. + else + sinc=sin(pi*q)/pi/q + end if + return + end function sinc + + subroutine fft2d(arr,ixfft,iyfft,dir) + + integer*4 ixfft,iyfft,dir + complex*8 arr(ixfft,iyfft) + integer*8 planf, plani + + if(dir.eq.dir)return + if(dir.eq.-1)then +! print *,'forward start',ixfft,iyfft +! call fftw2d_f77_create_plan(planf, ixfft, iyfft, -1, 8) +! call fftwnd_f77_one(planf, arr, work) +! call fftw_f77_destroy_plan(planf) + call sfftw_plan_dft_2d(planf,ixfft,iyfft,arr,arr,-1,64) + call sfftw_execute_dft(planf,arr,arr) + call sfftw_destroy_plan(planf) +! print *,'forward end' + end if + + if(dir.eq.1)then +! print *,'reverse start' +! call fftw2d_f77_create_plan(plani, ixfft, iyfft, +1, 8) +! call fftwnd_f77_one(plani, arr, work) +! call fftw_f77_destroy_plan(plani) + call sfftw_plan_dft_2d(plani,ixfft,iyfft,arr,arr,+1,64) + call sfftw_execute_dft(plani,arr,arr) + call sfftw_destroy_plan(plani) +! print *,'reverse end' + end if + + return + end subroutine fft2d + + subroutine median_similarity_intmask(phase,masks,scenemask,scr,len,lines,nigrams,psthresh,similarity) + !c median similarity of each ps point with neighbors in some distance range + ! compute median_similarity - for each ps pixel (>th), compute its phase similarity with + ! nearby PS and return the median of similarity measurements + + use omp_lib + implicit none + + real*4 scr(len,lines),similarity(len,lines),phase(len,lines,nigrams) + integer*1 masks(len,lines,nigrams),scenemask(len,lines) + real*4 rmin, rmax, psthresh, dist, cossim(20), temp_cossim + integer len,lines,i,j,ii,jj,p,nigrams,nvalidpair,irmin,irmax,neigh,k,boxsize + integer*4, allocatable :: iindex(:),jindex(:) + real*4, allocatable :: r(:) + + irmin=2 + irmax=50 + boxsize=2*irmax+1 + + ! set up spiral scanning + allocate (iindex(boxsize*boxsize),jindex(boxsize*boxsize),r(boxsize*boxsize)) + + call spiralscan(boxsize,iindex,jindex,r) + + ! how many are we starting with? + k=0 + do j=1,lines + do i=1,len + if(scr(i,j).ge.psthresh)k=k+1 + end do + end do +! print *,'Starting median pass with number of ps = ',k + + ! loop over all ps pixels and compute median similarity + !$omp parallel do private(j,k,p,neigh,cossim,nvalidpair,temp_cossim,dist,ii,jj) & + !$omp shared(lines,len,boxsize,iindex,jindex,r,irmin,irmax,scr,psthresh,nigrams,similarity,phase,masks,scenemask) + do i=1,len + do j=1,lines + similarity(i,j)=0. + if (scenemask(i,j)==0)then + cycle + end if + if(scr(i,j).ge.psthresh)then ! found an scr ps candidate + ! find neighbor ps + neigh=0 + cossim=0. + do k=1,boxsize*boxsize + if(neigh.lt.20)then ! only continue if less than 20 neighbors found + ii=i+iindex(k) + jj=j+jindex(k) + dist=r(k) + if(dist.ge.irmin.and.dist.le.irmax)then ! in ring to check + if(ii.ge.1.and.ii.le.len.and.jj.ge.1.and.jj.le.lines)then ! in bounds + + if(scr(ii,jj).ge.psthresh)then ! found a neighbor ps candidate + neigh=neigh+1 + if(neigh.le.20)then + nvalidpair = 0 + temp_cossim = 0.0 + !$omp parallel do private(p) shared(masks,phase,temp_cossim,nvalidpair,i,j,ii,jj) + do p=1,nigrams + if(masks(i,j,p).ne.0.and.masks(ii,jj,p).ne.0)then + temp_cossim=temp_cossim+cos(phase(i,j,p)-phase(ii,jj,p)) + nvalidpair=nvalidpair+1 + end if + end do + !$omp end parallel do + if (nvalidpair>1)then + cossim(neigh)=temp_cossim/nvalidpair + else + neigh=neigh-1 + end if + !cossim(neigh)=sum(cos(phase(i,j,:)-phase(ii,jj,:)))/nigrams + end if + end if ! end threshold test + end if ! end in-bounds test + end if ! end distance test + end if ! end less than 20 test + ! if(neigh.ge.20)exit + end do + ! similarity(i,j)=0. + if(neigh.ge.1)then ! if we found neighbor ps candidates + if(neigh.eq.1)then + similarity(i,j)=cossim(1) + else + call sort(neigh,cossim) + similarity(i,j)=cossim(neigh/2) + end if +! print *,i,j,neigh,similarity(i,j),cossim + end if + end if ! end initial candidate test + end do ! end line loop + end do ! end pixel loop + + return + end subroutine median_similarity_intmask + + subroutine median_similarity(phase,scenemask,scr,len,lines,nigrams,psthresh,similarity) + !c median similarity of each ps point with neighbors in some distance + !range + ! compute median_similarity - for each ps pixel (>th), compute its phase + ! similarity with + ! nearby PS and return the median of similarity measurements + + use omp_lib + implicit none + + real*4 scr(len,lines),similarity(len,lines),phase(len,lines,nigrams) + integer*1 scenemask(len,lines) + real*4 rmin, rmax, psthresh, dist, cossim(20), temp_cossim + integer len,lines,i,j,ii,jj,p,nigrams,nvalidpair,irmin,irmax,neigh,k,boxsize + integer*4, allocatable :: iindex(:),jindex(:) + real*4, allocatable :: r(:) + + irmin=2 + irmax=50 + boxsize=2*irmax+1 + + ! set up spiral scanning + allocate (iindex(boxsize*boxsize),jindex(boxsize*boxsize),r(boxsize*boxsize)) + + call spiralscan(boxsize,iindex,jindex,r) + + ! how many are we starting with? + k=0 + do j=1,lines + do i=1,len + if(scr(i,j).ge.psthresh)k=k+1 + end do + end do + + ! loop over all ps pixels and compute median similarity + !$omp parallel do private(j,k,p,neigh,cossim,nvalidpair,temp_cossim,dist,ii,jj) & + !$omp shared(lines,len,boxsize,iindex,jindex,r,irmin,irmax,scr,psthresh,nigrams,similarity,phase,scenemask) + do i=1,len + do j=1,lines + similarity(i,j)=0. + if (scenemask(i,j)==0)then + cycle + end if + if(scr(i,j).ge.psthresh)then ! found an scr ps candidate + ! find neighbor ps + neigh=0 + cossim=0. + do k=1,boxsize*boxsize + if(neigh.lt.20)then ! only continue if less than 20neighbors found + ii=i+iindex(k) + jj=j+jindex(k) + dist=r(k) + if(dist.ge.irmin.and.dist.le.irmax)then ! in ring to check + if(ii.ge.1.and.ii.le.len.and.jj.ge.1.and.jj.le.lines)then + ! in bounds + + if(scr(ii,jj).ge.psthresh)then ! found a neighbor ps candidate + neigh=neigh+1 + if(neigh.le.20)then + cossim(neigh)=sum(cos(phase(i,j,:)-phase(ii,jj,:)))/nigrams + end if + end if ! end threshold test + end if ! end in-bounds test + end if ! end distance test + end if ! end less than 20 test + end do + + if(neigh.ge.1)then ! if we found neighbor ps candidates + if(neigh.eq.1)then + similarity(i,j)=cossim(1) + else + call sort(neigh,cossim) + similarity(i,j)=cossim(neigh/2) + end if + end if + end if ! end initial candidate test + end do ! end line loop + end do ! end pixel loop + + return + end subroutine median_similarity + + subroutine all_similarity_intmask(phase,masks,scenemask,similarity,len,lines,nigrams,simthresh,allsimilarity) + ! all_similarity - return similarity for all pixels vis a vis ps points + ! only add newly discovered pixels, do not remove previous ones + + use omp_lib + implicit none + + real*4 allsimilarity(len,lines),similarity(len,lines),phase(len,lines,nigrams) + integer*1 masks(len,lines,nigrams),scenemask(len,lines) + real*4 rmin, rmax, dist, cossim(20),simthresh, temp_cossim + integer len,lines,i,j,ii,jj,p,nigrams,nvalidpair,irmin,irmax,neigh,boxsize,k + integer*4, allocatable :: iindex(:),jindex(:) + real*4, allocatable :: r(:) + + irmin=2 + irmax=50 + boxsize=2*irmax+1 + + ! set up spiral scanning + allocate (iindex(boxsize*boxsize),jindex(boxsize*boxsize),r(boxsize*boxsize)) + + call spiralscan(boxsize,iindex,jindex,r) + + ! loop over all pixels and find ps + !$omp parallel do private(i,k,p,neigh,nvalidpair,temp_cossim,cossim,dist,ii,jj) & + !$omp shared(lines,len,boxsize,iindex,jindex,r,irmin,irmax,similarity,psthresh,nigrams,allsimilarity,phase,masks,scenemask) + + do j=1,lines + if(mod(j,1000).eq.1)print *,'Computing similarity at line ',j + do i=1,len + allsimilarity(i,j)=0. + if(scenemask(i,j)==0)then + cycle + end if + if(similarity(i,j).ge.simthresh)then + allsimilarity(i,j)=similarity(i,j) + else + ! find neighbor ps + neigh=0 + cossim=0. + do k=1,boxsize*boxsize + if(neigh.lt.20)then !search until we get 20 neighbors + ii=i+iindex(k) + jj=j+jindex(k) + dist=r(k) + if(dist.ge.irmin.and.dist.le.irmax)then + if(ii.ge.1.and.ii.le.len.and.jj.ge.1.and.jj.le.lines)then ! in bounds + if(similarity(ii,jj).ge.simthresh)then ! found a neighbor + neigh=neigh+1 + if(neigh.le.20)then + nvalidpair=0 + temp_cossim=0.0 + !$omp parallel do private(p) shared(masks,phase,temp_cossim,nvalidpair,i,j,ii,jj) + do p=1,nigrams + if(masks(i,j,p).ne.0.and.masks(ii,jj,p).ne.0)then + temp_cossim=temp_cossim+cos(phase(i,j,p)-phase(ii,jj,p)) + nvalidpair = nvalidpair+1 + end if + end do + !$omp end parallel do + if (nvalidpair>1) then + cossim(neigh)=temp_cossim/nvalidpair + else + neigh=neigh-1 + end if + !cossim(neigh)=sum(cos(phase(i,j,:)-phase(ii,jj,:)))/nigrams + end if + end if ! end threshold test + end if ! end in-bounds test + end if ! end distance test + end if ! end test if reached 20 neighbors + end do + if(neigh.ge.1)then + if(neigh.eq.1)then + allsimilarity(i,j)=cossim(1) + else + allsimilarity(i,j)=sum(cossim(1:neigh))/neigh + end if + ! print *,neigh,i,j,similarity(i,j) + end if + end if ! finished looking for neighbors option + end do ! end of pixel loop + end do !end of line loop + !$omp end parallel do + + ! count number above threshold after computation + k=0 + do j=1,lines + do i=1,len + if(allsimilarity(i,j).ge.simthresh)k=k+1 + end do + end do + print *,'Number of similarity pixels found: ',k + + return + end subroutine all_similarity_intmask + + subroutine all_similarity(phase,scenemask,similarity,len,lines,nigrams,simthresh,allsimilarity) + ! all_similarity - return similarity for all pixels vis a vis ps points + ! only add newly discovered pixels, do not remove previous ones + + use omp_lib + implicit none + + real*4 allsimilarity(len,lines),similarity(len,lines),phase(len,lines,nigrams) + integer*1 scenemask(len,lines) + real*4 rmin, rmax, dist, cossim(20),simthresh, temp_cossim + integer len,lines,i,j,ii,jj,p,nigrams,nvalidpair,irmin,irmax,neigh,boxsize,k + integer*4, allocatable :: iindex(:),jindex(:) + real*4, allocatable :: r(:) + + irmin=2 + irmax=50 + boxsize=2*irmax+1 + + ! set up spiral scanning + allocate (iindex(boxsize*boxsize),jindex(boxsize*boxsize),r(boxsize*boxsize)) + + call spiralscan(boxsize,iindex,jindex,r) + + ! loop over all pixels and find ps + !$omp parallel do private(i,k,p,neigh,nvalidpair,temp_cossim,cossim,dist,ii,jj) & + !$omp shared(lines,len,boxsize,iindex,jindex,r,irmin,irmax,similarity,psthresh,nigrams,allsimilarity,phase,scenemask) + + do j=1,lines + if(mod(j,1000).eq.1)print *,'Computing similarity at line ',j + do i=1,len + allsimilarity(i,j)=0. + if(scenemask(i,j)==0)then + cycle + end if + if(similarity(i,j).ge.simthresh)then + allsimilarity(i,j)=similarity(i,j) + else + ! find neighbor ps + neigh=0 + cossim=0. + do k=1,boxsize*boxsize + if(neigh.lt.20)then !search until we get 20 neighbors + ii=i+iindex(k) + jj=j+jindex(k) + dist=r(k) + if(dist.ge.irmin.and.dist.le.irmax)then + if(ii.ge.1.and.ii.le.len.and.jj.ge.1.and.jj.le.lines)then ! in bounds + if(similarity(ii,jj).ge.simthresh)then ! found a neighbor + neigh=neigh+1 + if(neigh.le.20)then + cossim(neigh)=sum(cos(phase(i,j,:)-phase(ii,jj,:)))/nigrams + end if + end if ! end threshold test + end if ! end in-bounds test + end if ! end distance test + end if ! end test if reached 20 neighbors + end do + if(neigh.ge.1)then + if(neigh.eq.1)then + allsimilarity(i,j)=cossim(1) + else + allsimilarity(i,j)=sum(cossim(1:neigh))/neigh + end if + ! print *,neigh,i,j,similarity(i,j) + end if + end if ! finished looking for neighbors option + end do ! end of pixel loop + end do !end of line loop + !$omp end parallel do + + ! count number above threshold after computation + k=0 + do j=1,lines + do i=1,len + if(allsimilarity(i,j).ge.simthresh)k=k+1 + end do + end do + print *,'Number of similarity pixels found: ',k + + return + end subroutine all_similarity + + + SUBROUTINE SORT(N,RA) + integer N,IR,J,L,I + real*4 ra(N),rra + +! DIMENSION RA(N) + L=N/2+1 + IR=N +10 CONTINUE + IF(L.GT.1)THEN + L=L-1 + RRA=RA(L) + ELSE + RRA=RA(IR) + RA(IR)=RA(1) + IR=IR-1 + IF(IR.EQ.1)THEN + RA(1)=RRA + RETURN + ENDIF + ENDIF + I=L + J=L+L +20 IF(J.LE.IR)THEN + IF(J.LT.IR)THEN + IF(RA(J).LT.RA(J+1))J=J+1 + ENDIF + IF(RRA.LT.RA(J))THEN + RA(I)=RA(J) + I=J + J=J+J + ELSE + J=IR+1 + ENDIF + GO TO 20 + ENDIF + RA(I)=RRA + GO TO 10 + END + + subroutine spiralscan(boxsize,iindex,jindex,r) + + integer boxsize,x,y,n,count + integer*4 iindex(boxsize*boxsize),jindex(boxsize*boxsize) + integer*4, allocatable :: array(:,:),scan(:,:),xx(:) + real*4 r(boxsize*boxsize) + + allocate (array(boxsize,boxsize),scan(boxsize,boxsize),xx(boxsize)) + do i=1,boxsize + xx(i)=i-boxsize/2-1 + end do + + x=0 + y=1 + n=0 + count=boxsize; + do i = 1,count + x = x + 1 + array(x,y) = n + n = n + 1 + end do + do + count = count - 1 + do i = 1,count + y = y + 1 + array(x,y) = n + n = n + 1 + end do + do i = 1,count + x = x - 1 + array(x,y) = n + n = n + 1 + end do + if (n > boxsize*boxsize-1) exit + count = count - 1 + do i = 1,count + y = y - 1 + array(x,y) = n + n = n + 1 + end do + do i = 1,count + x = x + 1 + array(x,y) = n + n = n + 1 + end do + if (n > boxsize*boxsize-1) exit + end do + scan=boxsize*boxsize-array + +!c create list of indices in scan order + do i=1,boxsize + do j=1,boxsize + iindex(scan(i,j))=xx(i) + jindex(scan(i,j))=xx(j) + end do + end do + !c the distance between center point and each scanned pixels + do k = 1, boxsize*boxsize + r(k) = sqrt(real(iindex(k))**2.+real(jindex(k))**2.) + end do + + return + end subroutine spiralscan + +END PROGRAM cosine_sim_mask + diff --git a/kp_scripts/ps/make_simmasks_mask.py b/kp_scripts/ps/make_simmasks_mask.py new file mode 100644 index 0000000..341b6aa --- /dev/null +++ b/kp_scripts/ps/make_simmasks_mask.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 + +# make_simmasks_mask.py -- create ps mask for individual SLCs listed in geolist. + +# Included interferograms for each SLC are determined from the corresponding intlist for each. + +import sys +from datetime import datetime +import os +import math + +if len(sys.argv) < 3: + print ('Usage: make_simmasks_mask.py geolist length ') + sys.exit(1) + +geolist=sys.argv[1] +length=sys.argv[2] + +# Default Parameters +scenemask = 'none' +intmaskflag = '0' +psthresh='2' +simthresh='0.5' + +# Get optional input parameters +if len(sys.argv) > 3: + scenemask=sys.argv[3] +if len(sys.argv) > 4: + intmaskflag = sys.argv[4] +if len(sys.argv) > 5: + psthresh=sys.argv[5] +if len(sys.argv) > 6: + simthresh=sys.argv[6] + +##### ----- Run cosine_sim_mask for each SLC in geolist ----- ##### +# Import geolist +fgeos=open(geolist,'r') +geos=fgeos.readlines() +fgeos.close() + +# Run through each SLC +for line in geos: + words=line.split('/') + indate=words[-1][:8] + intlist = 'intlist_'+indate + maskfile = indate+'_merged_multi.mask' + + if scenemask != 'none': + # Create a temporary mask that is the intersection of the multilooked mask file for the SLC and the input scenemask + print(' Intersecting SLC mask with '+scenemask) + command = '$PROC_HOME/kp_scripts/int/int_simmask '+maskfile+' '+scenemask+' '+length+' '+maskfile+'_tempmask' + print(' '+command) + ret=os.system(command) + maskfile = maskfile+'_tempmask' + + allsimfile = 'pssim_'+indate + print('\n ##### ----- '+intlist+' ----- #####') + # find the ps and store in various formats + command="$PROC_HOME/kp_scripts/ps/cosine_sim_mask "+intlist+' '+length+' '+allsimfile+' '+maskfile+' '+intmaskflag+' '+psthresh+' '+simthresh+' _'+indate + print(' '+command) + ret=os.system(command) + + if scenemask != 'none': + ret=os.system('rm '+maskfile) diff --git a/kp_scripts/ps/make_simmasks_mask_parallel.py b/kp_scripts/ps/make_simmasks_mask_parallel.py new file mode 100644 index 0000000..ed87033 --- /dev/null +++ b/kp_scripts/ps/make_simmasks_mask_parallel.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 + +# make_simmasks_mask_parallel.py -- create ps mask for an input intlist, and many masks given optional input geolist + +import sys +from datetime import datetime +import os +import subprocess +import glob +import math + +if len(sys.argv) < 3: + print ('Usage: make_simmasks_mask_parallel.py length geolist ') + sys.exit(1) + +# Get input parameters +length=sys.argv[1] +geolist = sys.argv[2] + +# Optional input parameters +scenemask = 'none' +intmaskflag = '0' +psthresh='2' +simthresh='0.5' +if len(sys.argv) > 3: + scenemask = sys.argv[3] +if len(sys.argv) > 4: + intmaskflag = sys.argv[4] +if len(sys.argv) > 5: + psthresh=sys.argv[5] +if len(sys.argv) > 6: + simthresh=sys.argv[6] + +# 1. Open geolist +fgeo=open(geolist,'r') +geos=fgeo.readlines() +fgeo.close() + +# 2. determine which similarity masks still need to be made +geo_sim = [] +with open('geolist_for_sim','w') as fgeolist: + simmasks = glob.glob('all_similarity_mask_*') + for geo in geos: + words = geo.strip().split('/') + date = words[-1][:8] + name = 'all_similarity_mask_'+date + if name not in simmasks: + fgeolist.write(geo) + geo_sim.append(geo.rstrip()) +geofiles = 'geolist_for_sim' + +# If there are similarity masks to make, make them +if len(geo_sim)>0: + # 3. Create multiple lists of geolists to parallelize + command = '$PROC_HOME/util/splitintlist.py '+geofiles + print(command) + ret=os.system(command) + + # 4. Run make_simmasks_mask.py in parallel for each geolist generated in (3) + # NOTE: the splitintlist.py script assumes your outputs are named intlist###. Rather than generate a new script, I've just kept the outputs as intlist## and call those. They get deleted at the end of this script + list_geo = [] + for k in range(30): + if os.path.isfile('intlist'+str(k)): + list_geo.append('intlist'+str(k)) + + num=0 + simcommand=[] + prochome = os.getenv('PROC_HOME') + for geo in list_geo: + if os.path.getsize(geo.rstrip())>0: + command = '$PROC_HOME/kp_scripts/ps/make_simmasks_mask.py '+geo+' '+length+' '+scenemask+' '+intmaskflag+' '+psthresh+' '+simthresh + print(' '+command) + simcommand.append(subprocess.Popen([prochome+'/kp_scripts/ps/make_simmasks_mask.py',geo,length,scenemask,intmaskflag,psthresh,simthresh])) + num=num+1 + + for i in range(num): + simcommand[i].wait() + + print('\n All Similarity Masks Have Been Generated!') +else: + print('\n All Similarity Masks Have Already Been Generated!') + +command = 'rm geolist_for_sim' +print(command) +ret=os.system(command) + +for k in range(30): + if os.path.isfile('intlist'+str(k)): + command = 'rm intlist'+str(k) + ret=os.system(command) + diff --git a/kp_scripts/ps/psfilter_hz.py b/kp_scripts/ps/psfilter_hz.py new file mode 100644 index 0000000..36dc0b3 --- /dev/null +++ b/kp_scripts/ps/psfilter_hz.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 + +# psfilter.py -- create ps mask and interpolate ps from a second list + +import sys +from datetime import datetime +import os +import math + +if len(sys.argv) < 4: + print ('Usage: psfilter_hz.py interpolatelist length simmask') + sys.exit(1) + +interpolatelist=sys.argv[1] +length=sys.argv[2] +simmask=sys.argv[3] + + +# interpolate the files in the interpolatelist +finterp=open(interpolatelist,'r') +interp=finterp.readlines() +finterp.close() +for intfile in interp: + if os.path.isfile(intfile.strip()+'.interp')==False: + command="$PROC_HOME/ps/psinterp "+intfile.strip()+" "+simmask+" "+intfile.strip()+".interp "+length + print(command) + ret=os.system(command) diff --git a/kp_scripts/ps/psfilter_kp.py b/kp_scripts/ps/psfilter_kp.py new file mode 100644 index 0000000..702a2a4 --- /dev/null +++ b/kp_scripts/ps/psfilter_kp.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 + +# psfilter_kp.py -- create ps mask and interpolate ps from a second list + +import sys +from datetime import datetime +import os +import math + +if len(sys.argv) < 3: + print ('Usage: psfilter_kp.py interpolatelist length') + sys.exit(1) + +interpolatelist=sys.argv[1] +length=sys.argv[2] + + +# create simmask and interpolate the files in the interpolatelist +finterp=open(interpolatelist,'r') +interp=finterp.readlines() +finterp.close() + +for intfile in interp: + if os.path.isfile(intfile.strip()+'.interp')==False: + simmask='temp_mask' + # Make the mask for the interferogram + words=intfile.strip().split('_') + date1=words[0] + date2=words[1][:8] + mask1='all_similarity_mask_'+date1 + mask2='all_similarity_mask_'+date2 + command='$PROC_HOME/kp_scripts/int/int_simmask '+mask1+' '+mask2+' '+length + print(command) + ret=os.system(command) + command="$PROC_HOME/ps/psinterp "+intfile.strip()+" "+simmask+" "+intfile.strip()+".interp "+length + print(command) + ret=os.system(command) diff --git a/kp_scripts/sbas/.nfs000000000001fc5e00000003 b/kp_scripts/sbas/.nfs000000000001fc5e00000003 new file mode 100644 index 0000000..64b7863 Binary files /dev/null and b/kp_scripts/sbas/.nfs000000000001fc5e00000003 differ diff --git a/kp_scripts/sbas/intlist_from_sbas.py b/kp_scripts/sbas/intlist_from_sbas.py new file mode 100644 index 0000000..da1c4f4 --- /dev/null +++ b/kp_scripts/sbas/intlist_from_sbas.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 + +# intlist_from_sbas.py - write out intlist, either full set or subset the input sbaslist to include only interferograms that contain the optional input date + +import sys +import string +import os +import math + +if len(sys.argv) < 2: + print ('Usage: intlist_from_sbas.py sbas_list ') + sys.exit(1) + +sbaslist=sys.argv[1] +indate='none' +if len(sys.argv) > 2: + indate = sys.argv[2] + +# Output sbaslist and intlist +intlist=sbaslist.replace('sbas','int') +if indate != 'none': + intlist = 'intlist_'+indate +fintlist=open(intlist,'w') + +# Read in full sbaslist +with open(sbaslist,'r') as fsbas: + sbas=fsbas.readlines() + +for line in sbas: + words=line.split() + primary=words[0] + secondary=words[1] + + # get a short names for primary and secondary files + first=primary.find('20') + primaryname=primary[first:first+8] + first=secondary.find('20') + secondaryname=secondary[first:first+8] + + # If an input date was given, check to see if it should go in the list + if indate != 'none': + if primaryname != indate: + if secondaryname != indate: + continue + + intfile=primaryname+'_'+secondaryname+'.int' + + fintlist.write(intfile+'\n') + +fintlist.close() + +sys.exit() + diff --git a/kp_scripts/sbas/not_used/find_pixel_groups b/kp_scripts/sbas/not_used/find_pixel_groups new file mode 100644 index 0000000..b34ba8d Binary files /dev/null and b/kp_scripts/sbas/not_used/find_pixel_groups differ diff --git a/kp_scripts/sbas/not_used/find_pixel_groups.f90 b/kp_scripts/sbas/not_used/find_pixel_groups.f90 new file mode 100644 index 0000000..c74a2bb --- /dev/null +++ b/kp_scripts/sbas/not_used/find_pixel_groups.f90 @@ -0,0 +1,426 @@ +! Determine valid geocoded slcs and pixels within the scene. +! This will result in two new geolists: +! geolist_consistent = list of SLCs that share consistent coverage. Script +! maximizes number of SLCs with the same coverage (could be improved to +! optimize temporal & spatial coverage) +! +! geolist_culled = list of SLCs that cover at least thresh% of the valid +! study area. The valid area is the union of all scenes' coverage (determined +! from the input masks). The default threshold is 30% of valid. +! +! The script also outputs summation maps (The number of valid acquisitions at +! each pixel): +! sum_coverage_geolist_ml (based on all geocoded slcs that have been +! processed) +! +! sum_coverage_geolist_consistent (based on geolist_consistent) +! +! sum_coverage_geolist_culled (based on geolist_culled) +! +! The script will also output the corresponding binary masks: +! scene_mask_ml = indicates where sum_coverage_geolist_ml > 1 +! +! scene_mask_consistent = indicates where sum_coverage_geolist_consistent +! equals the number of consistent SLCs +! +! scene_mask_culled = indicates where sum_coverage_geolist_culled > 1 + +! NOTE: the script could be improved by excluding valid scene edges (e.g. far +! range or close range), since these are wildly inconsistent and we don't expect +! the accuracy at these locations to be very high. + + +! Compile with: +! gfortran -o determine_valid determine_valid.f90 -fopenmp -lrt -lpthread + +PROGRAM find_pixel_groups + use omp_lib + + IMPLICIT none + + !!!!! SPECIFICATIONS !!!!! + + ! INPUTS + INTEGER:: nslc + INTEGER*8::nr,naz + CHARACTER(100)::geolist,scenemask + + ! ERROR/STATUS CHECKS/COUNTING VARIABLES + INTEGER::k,j,stat,ierr,stat2,ios + INTEGER,DIMENSION(13)::statb + CHARACTER(100)::str + + ! READING IN INPUTS LISTS AND DEFINING OTHER VARIABLES + CHARACTER(100),DIMENSION(:),ALLOCATABLE::geonames + + ! DATA ARRAYS FOR INPUT MASKS + INTEGER,DIMENSION(:,:,:),ALLOCATABLE::scenemasks + INTEGER*1,DIMENSION(:,:),ALLOCATABLE::mask + INTEGER,DIMENSION(:,:),ALLOCATABLE::mask_count + + ! LOADING IN THE MASKS + CHARACTER(100)::strgeo,strmask + + ! ARRAYS FOR SUMMING VALID ACQUISITIONS + INTEGER,DIMENSION(:,:),ALLOCATABLE::sum_full,sum_consistent,sum_culled + REAL,DIMENSION(:,:),ALLOCATABLE::temp_array + + ! ARRAYS FOR IDENTIFYING POTENTIALLY VALID PIXELS (MASKS) + INTEGER,DIMENSION(:,:),ALLOCATABLE::valid_full,valid_consistent,valid_culled + INTEGER*8::nvalid_full,nvalid_consistent,nvalid_culled + + ! FOR DETERMINING COVERAGE GROUPS + INTEGER,DIMENSION(:),ALLOCATABLE::pixel_counts,counts,counts_groups + REAL,DIMENSION(:),ALLOCATABLE::prct_of_valid,bins,bins_groups + REAL::dp + INTEGER::nslc_consistent,nslc_culled + INTEGER, DIMENSION(1) :: max_idx + INTEGER::n,idx_consistent,counter,n_groups + INTEGER,DIMENSION(:),ALLOCATABLE::index_consistent,index_culled + + ! NAMES FOR OUTPUT FILES + CHARACTER(100)::geolist_consistent,geolist_culled + CHARACTER(100)::sum_full_out,sum_consistent_out,sum_culled_out + CHARACTER(100)::scenemask_full_out,scenemask_consistent_out,scenemask_culled_out + + ! ARRAYS FOR OUTPUT GEOLISTS + CHARACTER(100),DIMENSION(:),ALLOCATABLE::geonames_consistent,geonames_culled + + ! MASK OUTPUT ARRAYS + INTEGER*1,DIMENSION(:,:),ALLOCATABLE::mask_full,mask_consistent,mask_culled + + + !!!!! EXECUTIONS !!!!! + ! check to see if proper number of arguments were provided + IF(iargc().lt.4)THEN + WRITE(*,*)'usage: find_pixel_groups geolist nslc nr naz ' + STOP + END IF + + ! read in arguments and convert to appropriate type (where necessary) + CALL getarg(1,geolist) + CALL getarg(2,str) + READ(str,*)nslc + CALL getarg(3,str) + READ(str,*)nr + CALL getarg(4,str) + READ(str,*)naz + + scenemask='none' + IF(iargc().gt.4)THEN + CALL getarg(5,scenemask) + END IF + + ! Allocate the list of .geo files and the scenemasks array + ALLOCATE(geonames(nslc)) + ALLOCATE(scenemasks(nr,naz,nslc)) + + ! Read in the geolist + OPEN(unit=11,FILE=geolist,STATUS='old') + READ(11,'(A)',end=110,IOSTAT=stat2)geonames + 110 continue + CLOSE(11) + + !!!!! ----- READ IN ALL MASKS AND SUMMARIZE ----- !!!!! + + scenemasks=0 + ! Read in the scene masks + !$OMP parallel do private(strgeo,strmask,mask) & + !$OMP shared(geonames,scenemasks) + DO k=1,nslc + ALLOCATE(mask(nr,naz)) + strgeo = trim(adjustl(geonames(k))) + strmask = Replace_Text(strgeo,'.geo','.mask') + + ! Read the corresponding bytemask file + open(unit=k+10,file=strmask,access='stream') + read(k+10)mask + CLOSE(k+10) + scenemasks(:,:,k)=mask + + DEALLOCATE(mask) + END DO + !$OMP end parallel do + + !!!!! ----- FIND UNIQUE PIXELS ----- !!!!! + + + ! Get the sum of acquisitions at each pixel + ALLOCATE(sum_full(nr,naz)) + sum_full=0 + sum_full=sum(abs(scenemasks),dim=3) + + ! Summarize Potentially-Valid Pixels + ALLOCATE(valid_full(nr,naz)) + valid_full = 0 + !$OMP parallel do private(j) shared(valid_full,sum_full) + DO k=1,nr + DO j=1,naz + if (sum_full(k,j) >= 1) then + valid_full(k,j) = 1 + end if + END DO + END DO + !$OMP end parallel do + + nvalid_full = sum(valid_full(:,:)) + print *,' Number of Potentially Valid Pixels Across All Scenes: ',nvalid_full + + + !!!!! ----- FIND COVERAGE GROUPS ----- !!!!! + + ! Determine the number of valid pixels in each scene + ALLOCATE(pixel_counts(nslc)) + !$OMP parallel do private(mask_count) shared(pixel_counts,scenemasks) + DO k=1,nslc + ALLOCATE(mask_count(nr,naz)) + mask_count = scenemasks(:,:,k) + pixel_counts(k) = sum(abs(mask_count)) + DEALLOCATE(mask_count) + END DO + !$OMP end parallel do + + ! Determine the percentage of valid pixels, relative to the number of pixels + ! covered by at least one scene + ALLOCATE(prct_of_valid(nslc)) + prct_of_valid = real(pixel_counts)/real(nvalid_full)*100.0 + + ! Bin the coverage by 0.5% windows + dp = 0.5 !step size for percentage bins + n = int((100.0 - dp) / dp) + 1 ! number of bins + + ALLOCATE(bins(n)) + ! Fill the bins with percentage + DO k=1,n + bins(k) = (k-1) * dp + END DO + + ! Find the number of scenes that correspond to each percentage bin + ALLOCATE(counts(n)) + counts=0 + DO k=1,nslc + DO j=1,n + IF (j/=n) THEN + IF (prct_of_valid(k).ge.bins(j).and.prct_of_valid(k).lt.bins(j)+dp)THEN + counts(j)=counts(j)+1 + exit + END IF + ELSE + IF(prct_of_valid(k).ge.bins(j).and.prct_of_valid(k).le.bins(j)+dp)THEN + counts(j)=counts(j)+1 + END IF + END IF + END DO + END DO + + ! List Coverage Groups + print*,'' + print*,' VALID COVERAGE GROUPS' + n_groups = 0 + DO k=1,n + if (counts(k)>0.and.bins(k).ge.thresh) then + print*,'' + print*,' Percentage Bin: ',bins(k),' - ',bins(k)+dp + print*,' Number of SLCs: ',counts(k) + n_groups = n_groups+1 + end if + END DO + print*,'' + print*,' Number of groups: ',n_groups + + ! Capture Coverage Groups with valid coverage greater than input threshold + ALLOCATE(bins_groups(n_groups)) + ALLOCATE(counts_groups(n_groups)) + counter=1 + DO k=1,n + if (counts(k)>0.and.bins(k).ge.thresh) then + bins_groups(counter) = bins(k) + counts_groups(counter) = counts(k) + counter=counter+1 + end if + END DO + + print*,bins_groups + + !!!!! ----- CREATE ARRAYS AND GEOLIST FOR CONSISTENT COVERAGE ----- !!!!! + + ! Find the percentage bin corresponding to the maximum number of SLCs + nslc_consistent = maxval(counts) + max_idx = maxloc(counts) + idx_consistent = max_idx(1) + + print*,'' + print*,' Consistent Coverage Group: ',bins(idx_consistent),' - ',bins(idx_consistent)+dp + print*,' Number of SLCS: ',nslc_consistent + + ! Consistent Geolist + geolist_consistent = 'geolist_consistent' + ALLOCATE(geonames_consistent(nslc_consistent)) + ALLOCATE(index_consistent(nslc_consistent)) + counter=1 + DO k=1,nslc + if (idx_consistent/=n) THEN + if (prct_of_valid(k).ge.bins(idx_consistent).and.prct_of_valid(k).lt.bins(idx_consistent)+dp) then + geonames_consistent(counter) = geonames(k) + index_consistent(counter) = k + counter=counter+1 + end if + else + if (prct_of_valid(k).ge.bins(n).and.prct_of_valid(k).le.bins(n)+dp) then + geonames_consistent(counter) = geonames(k) + index_consistent(counter) = k + counter=counter+1 + end if + end if + END DO + + + ! Consistent Summation and Mask + ALLOCATE(sum_consistent(nr,naz)) + ALLOCATE(valid_consistent(nr,naz)) + + sum_consistent = sum(abs(scenemasks(:,:,index_consistent)),dim=3) + valid_consistent = 0 + !$OMP parallel do private(j) shared(sum_consistent,valid_consistent) + DO k=1,nr + DO j=1,naz + if (sum_consistent(k,j) == nslc_consistent) then + valid_consistent(k,j) = 1 + end if + END DO + END DO + !$OMP end parallel do + + !!!!! ----- CREATE ARRAYS AND GEOLIST FOR CULLED COVERAGE ----- !!!!! + + ! Find the percentage bin corresponding to the first group with coverage > + ! input threshold + nslc_culled = sum(counts_groups) + + print*,'' + print*,' Culled Coverage Minimum Valid: ',bins_groups(1) + print*,' Number of SLCS: ',nslc_culled + + + ! Culled Geolist + geolist_culled = 'geolist_culled' + ALLOCATE(geonames_culled(nslc_culled)) + ALLOCATE(index_culled(nslc_culled)) + counter=1 + DO k=1,nslc + if (prct_of_valid(k).ge.bins_groups(1)) then + geonames_culled(counter) = geonames(k) + index_culled(counter) = k + counter=counter+1 + end if + END DO + + ! Culled Summation and Mask + ALLOCATE(sum_culled(nr,naz)) + ALLOCATE(valid_culled(nr,naz)) + + sum_culled = sum(abs(scenemasks(:,:,index_culled)),dim=3) + valid_culled = 0 + !$OMP parallel do private(j) shared(sum_culled,valid_culled) + DO k=1,nr + DO j=1,naz + if (sum_culled(k,j)>=2) then + valid_culled(k,j) = 1 + end if + END DO + END DO + !$OMP end parallel do + + !!!!! ----- MAKE THE OUTPUT MASKS ----- !!!!! + ALLOCATE(mask_full(nr,naz),mask_consistent(nr,naz),mask_culled(nr,naz)) + !$OMP parallel do private(j) & + !$OMP shared(valid_full,valid_consistent,valid_culled,mask_full,mask_consistent,mask_culled) + DO k=1,nr + DO j=1,naz + if (valid_full(k,j)==1) then + mask_full(k,j)=-1 + end if + if (valid_consistent(k,j)==1) then + mask_consistent(k,j)=-1 + end if + if (valid_culled(k,j)==1) then + mask_culled(k,j) = -1 + end if + END DO + END DO + !$OMP end parallel do + + + !!!!! ----- WRITE OUT GEOLISTS, SUMS, AND MASKS ----- !!!!! + print *,'' + print *,' Writing out files...' + + ! Geolists + OPEN(28,FILE=geolist_culled,status='unknown',ACTION='write') + DO k=1,nslc_culled + WRITE(28,'(A)') trim(geonames_culled(k)) + END DO + CLOSE(28) + + OPEN(28,FILE=geolist_consistent,status='unknown',ACTION='write') + DO k=1,nslc_consistent + WRITE(28,'(A)') trim(geonames_consistent(k)) + END DO + CLOSE(28) + + ! Summation Arrays + sum_full_out = 'num_scenes_full.out' + sum_consistent_out = 'num_scenes_consistent.out' + sum_culled_out = 'num_scenes_culled.out' + + OPEN(28,FILE=sum_full_out,FORM='unformatted',ACCESS='stream') + WRITE(28)sum_full + CLOSE(28) + + OPEN(28,FILE=sum_consistent_out,FORM='unformatted',ACCESS='stream') + WRITE(28)sum_consistent + CLOSE(28) + + OPEN(28,FILE=sum_culled_out,FORM='unformatted',ACCESS='stream') + WRITE(28)sum_culled + CLOSE(28) + + ! Masks + scenemask_full_out = 'mask_full' + scenemask_consistent_out = 'mask_consistent' + scenemask_culled_out = 'mask_culled' + + OPEN(28,FILE=scenemask_full_out,FORM='unformatted',ACCESS='stream') + WRITE(28)mask_full + CLOSE(28) + + OPEN(28,FILE=scenemask_consistent_out,FORM='unformatted',ACCESS='stream') + WRITE(28)mask_consistent + CLOSE(28) + + OPEN(28,FILE=scenemask_culled_out,FORM='unformatted',ACCESS='stream') + WRITE(28)mask_culled + CLOSE(28) + + PRINT*,'Data written' + + + !!!!! FUNCTIONS !!!!! + + CONTAINS + + ! ------------------ Replace_Text ----------------------- ! + FUNCTION Replace_Text (s,text,rep) RESULT(outs) + CHARACTER(*) :: s,text,rep + CHARACTER(LEN(s)+100) :: outs ! provide outs with extra 100 char len + INTEGER :: i, nt, nr + + outs = s ; nt = LEN_TRIM(text) ; nr = LEN_TRIM(rep) + DO + i = INDEX(outs,text(:nt)) ; IF (i == 0) EXIT + outs = outs(:i-1) // rep(:nr) // outs(i+nt:) + END DO + END FUNCTION Replace_Text + + +END PROGRAM find_pixel_groups diff --git a/kp_scripts/sbas/not_used/multi_sbas_setup.py b/kp_scripts/sbas/not_used/multi_sbas_setup.py new file mode 100644 index 0000000..af7f37f --- /dev/null +++ b/kp_scripts/sbas/not_used/multi_sbas_setup.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 + +# create a list of sbas pairs for each geolist in listofgeolists (nearest and nearest+maxtb) + +import sys +import subprocess +from datetime import datetime +import os + +if len(sys.argv) < 3: + print('Usage: multi_sbas_setup.py listofgeolists sbas_suffix') + sys.exit(1) + +listofgeolists = sys.argv[1] +sbassuff = sys.argv[2] +#maxtemporal=int(sys.argv[3]) + +# open listofgeolists +fgeo = open(listofgeolists,'r') +geos = fgeo.readlines() +fgeo.close() + +# Make sbas lists +for geolist in geos: + + geolist2 = geolist.replace('.txt','').strip() + + # get names of files + sbas_list = geolist2.replace('geolist','sbas') + sbas_list = sbas_list+sbassuff + + suff = geolist2.replace('geolist','') + + # Run the sbas setup + command = '$PROC_HOME/sbas/sbas_setup_kp.py '+sbas_list+' '+geolist.strip()+' '+suff + print(command) + ret=os.system(command) + +print ('#### ---- ALL SBAS STRUCTURES WRITTEN ----- #####') diff --git a/kp_scripts/sbas/not_used/multi_sbas_setup_takespath.py b/kp_scripts/sbas/not_used/multi_sbas_setup_takespath.py new file mode 100644 index 0000000..5ef119d --- /dev/null +++ b/kp_scripts/sbas/not_used/multi_sbas_setup_takespath.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 + +# Turn an sbaslist into an intlist with full path to intfiles + +import sys +import subprocess +from datetime import datetime +import os + +if len(sys.argv) < 2: + print('Usage: make_intlist_from_sbas.py sbaslist ') + sys.exit(1) + +sbaslist = sys.argv[1] +path2intfiles='' +if len(sys.argv) > 2: + path2intfiles = sys.argv[2] + + +##### ----- 1. USE SBASLIST TO MAKE THE INTLIST ----- ##### +fsbas = open(sbaslist,'r') +sbas = fsbas.readlines() +fsbas.close() + +intlistname = sbaslist.replace('sbas','int') +fint = open(intlistname,'w') +for line in sbas: + words = line.strip().split(' ') + date1 = words[0].split('/')[-1][:8] + date2 = words[1].split('/')[-1][:8] + + intfile = path2intfiles+date1+'_'+date2+'.int' + fint.write(intfile+'\n') + +print(intlistname+' written') +print('number of interferograms in list: '+str(len(sbas))+'\n') +fint.close() + + diff --git a/kp_scripts/sbas/not_used/run_sbas_multi.py b/kp_scripts/sbas/not_used/run_sbas_multi.py new file mode 100644 index 0000000..9bf63a5 --- /dev/null +++ b/kp_scripts/sbas/not_used/run_sbas_multi.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 + +# Organize files and run sbas_multi.f90 on interferograms from many lists + +import sys +import subprocess +from datetime import datetime +import os + +if len(sys.argv) < 8: + print('Usage: run_sbas_multi.py path2unwfiles sbaslist geolist path2geolists sbas_suffix len ref_locs') + sys.exit(1) + +path2unwfiles = sys.argv[1] +sbaslist = sys.argv[2] +geolist = sys.argv[3] +path2geolists = sys.argv[4] +suff = sys.argv[5] +length = int(sys.argv[6]) +ref_locs=sys.argv[7] + +##### ----- 1. USE SBASLIST TO MAKE THE UNWLIST ----- ##### +fsbas = open(sbaslist,'r') +sbas = fsbas.readlines() +fsbas.close() + +nunwfiles = len(sbas) +funw = open('unwlist_full','w') +for line in sbas: + words = line.strip().split(' ') + date1 = words[0].split('/')[-1][:8] + date2 = words[1].split('/')[-1][:8] + + unwfile = path2unwfiles+date1+'_'+date2+'.unw' + funw.write(unwfile+'\n') + +print('unwlist_full written') +print('number of unwrapped interferograms in list: '+str(nunwfiles)+'\n') +funw.close() + +##### ----- 2. READ INPUT GEOLIST AND STRIP OF PATH ----- ##### +fgeo = open(geolist,'r') +geo = fgeo.readlines() +fgeo.close() + +nslc = len(geo) +fgeostrip = open('geolist_strip','w') +for line in geo: + geoname = line.strip().split('/')[-1] + + fgeostrip.write(geoname+'\n') +fgeostrip.close() + +print('Geolist for input to sbas written') +print('Number of SLCs: '+str(nslc)+'\n') + +##### ----- 3. CREATE LISTOFUNWLISTS ----- ##### +command = 'ls '+path2geolists+'geolist_* |cat> listofgeolists' +print(command) +ret=os.system(command) + +fgeolists = open('listofgeolists','r') +geolists = fgeolists.readlines() +fgeolists.close() + +nunwlists = 0 +funwlists = open('listofunwlists','w') +for line in geolists: + if ((line.strip() != path2geolists+'geolist_full') and (line.strip() != path2geolists+'geolist_all')): + sbas_name = line.strip().replace('geolist','sbas')+'_'+suff + unwlist = line.strip().replace('geolist','unwlist') + + funwlists.write(unwlist+'\n') + nunwlists+=1 + + fsbas = open(sbas_name,'r') + sbaslist = fsbas.readlines() + fsbas.close() + + funw = open(unwlist,'w') + for sbas in sbaslist: + words = sbas.strip().split(' ') + if len(words)>0: + date1 = words[0].split('/')[-1][:8] + date2 = words[1].split('/')[-1][:8] + + unwfile = path2unwfiles+date1+'_'+date2+'.unw' + funw.write(unwfile+'\n') + funw.close() +funwlists.close() + +print('Unwlists and listofunwlists written!') +print('Number of unwlists: '+str(nunwlists)+'\n') + + +##### ----- 4. RUN SBAS_MULTI ----- ##### +print('Running SBAS!') +command = '$PROC_HOME/sbas/sbas_multi unwlist_full '+str(nunwfiles)+' '+str(nslc)+' '+str(length)+' listofunwlists '+str(nunwlists)+' ref_locs' +print(command) +ret=os.system(command) +print('SBAS COMPLETE!') + diff --git a/kp_scripts/sbas/not_used/sbas_multi b/kp_scripts/sbas/not_used/sbas_multi new file mode 100644 index 0000000..26be78e Binary files /dev/null and b/kp_scripts/sbas/not_used/sbas_multi differ diff --git a/kp_scripts/sbas/not_used/sbas_multi.f90 b/kp_scripts/sbas/not_used/sbas_multi.f90 new file mode 100644 index 0000000..2a35c3b --- /dev/null +++ b/kp_scripts/sbas/not_used/sbas_multi.f90 @@ -0,0 +1,724 @@ +! Run SBAS on unique pixel groups +! Each has own set of files that are read in sequentially +! Incomplete velocity solutions are interpolated to the full time series only between the +! First and last valid dates (i.e. no extrapolation) + +! Compile with: +! gfortran -o sbas_multi sbas_multi.f90 svd.f90 -fopenmp -lrt -lpthread + +PROGRAM sbas_multi + use omp_lib + + IMPLICIT none + + !!!!! SPECIFICATIONS !!!!! + + ! INPUTS + INTEGER:: ncells,nslc,nunwlists + INTEGER*8::nr + CHARACTER(100)::filelist,str,listofunwlists,ref_locs_file + + ! ERROR/STATUS CHECKS/COUNTING VARIABLES + INTEGER::i,k,kk,counts,j,stat,ierr,stat2,ios + INTEGER,DIMENSION(13)::statb + + ! READING IN INPUTS LISTS AND DEFINING OTHER VARIABLES + INTEGER::n_refs + INTEGER*8::naz + INTEGER,DIMENSION(:,:),ALLOCATABLE::ref_locs + CHARACTER(100),DIMENSION(:),ALLOCATABLE::unwnames,cells + + ! DATA ARRAYS FOR SET OF ALL INTERFEROGRAMS + INTEGER*1,DIMENSION(:,:,:),ALLOCATABLE::masks + REAL,DIMENSION(:,:,:),ALLOCATABLE::phase,amps + + ! ARRAYS FOR SUMMARIZING/REDUCING FULL DATA SET + INTEGER,DIMENSION(:,:),ALLOCATABLE::npts,idxf,idxl + REAL,DIMENSION(:),ALLOCATABLE::phase_ref,disp + REAL,DIMENSION(:,:),ALLOCATABLE::amp,stacktime,stack + + ! FOR LOADING FULL TIME SERIES INFORMATION (ARRAYS) + REAL,DIMENSION(:),ALLOCATABLE::timedeltas,cumsumf + REAL,DIMENSION(:,:),ALLOCATABLE::deltime + CHARACTER(100),DIMENSION(:),ALLOCATABLE::geolist + + ! DATE VARIABLES + CHARACTER(8)::date1,date2 + INTEGER :: year1, month1, day1 + INTEGER :: year2, month2, day2 + INTEGER :: days1, days2, days_between + + ! LOADING IN THE UNWRAPPED INTERFEROGRAMS, AMPLITUDES, AND MASKS; FIND REFERENCE PHASE AT EACH INTERFEROGRAM + INTEGER*1,DIMENSION(:,:),ALLOCATABLE::mask + CHARACTER(100)::strint,stramp,strmask + REAL,DIMENSION(:,:),ALLOCATABLE::dat + + ! FULL SOLUTION + REAL,DIMENSION(:,:,:),ALLOCATABLE::velocity + + ! SPECIFIC TO EACH PIXEL GROUP + CHARACTER(100),DIMENSION(:),ALLOCATABLE::unwlist,geolist_temp + CHARACTER(100)::geolist_name,Tmstr,timedeltas_file,pixel_file + INTEGER::nunw_temp,nvalid,npix,ndays_temp,tempday,idx + INTEGER,DIMENSION(:,:),ALLOCATABLE::pixels + REAL,DIMENSION(:),ALLOCATABLE::timedeltas_temp,cumsumt,deltime_temp,temp1 + REAL,DIMENSION(:,:),ALLOCATABLE::Tm,Tminv,phase_temp,amp_temp,vel_temp + + + !!!!! EXECUTIONS !!!!! + ! check to see if proper number of arguments were provided + IF(iargc().lt.6)THEN + WRITE(*,*)'usage: sbas_multi filelist nunwfiles nslcs len listofunwlists nunwlists )' + STOP + END IF + + ! read in arguments and convert to appropriate type (where necessary) + CALL getarg(1,filelist) + CALL getarg(2,str) + READ(str,*)ncells + CALL getarg(3,str) + READ(str,*)nslc + CALL getarg(4,str) + READ(str,*)nr + CALL getarg(5,listofunwlists) + CALL getarg(6,str) + READ(str,*)nunwlists + + ! allocate the list of unwrapped files and the reference pixel locations array + ALLOCATE(cells(ncells),ref_locs(2,10000000)) + ALLOCATE(unwnames(nunwlists)) + + ! read in the list of unwrapped files + OPEN(UNIT=11,FILE=filelist,STATUS='old') + READ(11,'(A)',end=10,IOSTAT=stat)cells + 10 continue + CLOSE(11) + + ! Read in the first unwrapped interferogram name in order to determine the number of azimuth lines, given the known number of range lines + OPEN(UNIT=21,FILE=trim(adjustl(cells(1))),FORM='unformatted',ACCESS='direct',RECL=nr*8) + ierr=fstat(21,statb) + naz=statb(8)/8/nr + WRITE(*,*)' Lines in file: ',naz + CLOSE(21) + + ! Read in the list of unwlists + OPEN(unit=11,FILE=listofunwlists,STATUS='old') + READ(11,'(A)',end=110,IOSTAT=stat2)unwnames + 110 continue + CLOSE(11) + + + ! Set list of reference points, default to scene center if no input file is given + n_refs=1 + ref_locs(1,1)=nr/2 + ref_locs(2,1)=naz/2 + if(iargc().ge.7)then + CALL getarg(7,str) + READ(str,*)ref_locs_file + open(unit=13,file=ref_locs_file) + do i=1,10000000 + read(13,*,end=99)ref_locs(:,i) + end do + 99 n_refs=i-1 + CLOSE(13) + end if + print *,'Number of reference points: ',n_refs + + ! Now that we know some additional parameters, allocate some of the arrays we will need + ALLOCATE(phase(nr,naz,ncells),amps(nr,naz,ncells),masks(nr,naz,ncells)) + ALLOCATE(phase_ref(ncells), disp(nr)) + + ! Set values in arrays equal to zero for now + phase(:,:,:)=0. + phase_ref = 0. + amps=0. + masks(:,:,:)=0 + + + !!!!! READ IN THE SBAS PARAMETERS NEEDED FOR FULL TIME SERIES !!!!! + + ! Create an array with data from deltime.out (currently hard-coded) + ALLOCATE(deltime(4,ncells)) + OPEN(13,FILE='deltime.out',STATUS='old') + DO i=1,ncells + READ(13,*,IOSTAT=stat)deltime(:,i) + END DO + CLOSE(13) + + ! Read time from slc to slc (i.e. temporal baseline between nearest neighbor scenes + ALLOCATE(timedeltas(nslc-1)) + open(13,file='timedeltas.out',status='old') + read(13,*)timedeltas + close(13) + + ! Make a cumulative sum of the timedeltas array for use during SBAS + ALLOCATE(cumsumf(nslc-1)) + do i=1,nslc-1 + cumsumf(i) = sum(timedeltas(1:i)) + end do + + ! Read in Geolist to get the first date of the time series + ALLOCATE(geolist(nslc)) + open(11,file='geolist',status='old') + read(11,'(A)',end=112,IOSTAT=stat2)geolist + 112 continue + close(11) + + ! Date of first SLC acquisition + date1 = trim(adjustl(geolist(1))) + READ(date1(1:4),*)year1 + READ(date1(5:6),*)month1 + READ(date1(7:8),*)day1 + + ! Convert dates to days since a reference date (e.g., 1 January 1900) + days1 = date_to_days(year1, month1, day1) + + + ! read in unwrapped igrams using parallel computing + !$OMP parallel do private(k,kk,dat,strint,stramp,strmask,mask,counts) & + !$OMP shared(cells,ref_locs,deltime,nr,naz,ncells,phase,amps,phase_ref,masks) + DO i=1,ncells + ! allocate dat, which appears to be a “temporary” variable holding the unwrapped data + allocate(dat(nr*2,naz)) + allocate(mask(nr,naz)) + + ! get the correct names corresponding to the files that will be read in + strint=trim(adjustl(cells(i))) + stramp=Replace_Text(strint,'.unw','.amp') + strmask=Replace_Text(strint,'.unw','.mask') + + ! open theinterferogram file, in binary format, directly. Requires conversion for reading. + OPEN(UNIT=i+10,FILE=strint,FORM='unformatted',ACCESS='direct',RECL=2*nr*naz*4,CONVERT= 'LITTLE_ENDIAN') + READ(i+10,rec=1)dat + CLOSE(i+10) + ! read the correct values from dat to get phase, and put in phase array + phase(:,:,i)=dat((nr+1):2*nr,:) + + ! Read the corresponding amplitude file, same way as the unwrapped file + OPEN(UNIT=i+10,FILE=stramp,FORM='unformatted',ACCESS='direct',RECL=2*nr*naz*4,CONVERT= 'LITTLE_ENDIAN') + READ(i+10,rec=1)dat + CLOSE(i+10) + ! read the correct values from dat to get the amplitude powers, put in amps array + amps(:,:,i)=dat(1:2*nr-1:2,:)**2+dat(2:2*nr:2,:)**2 + + ! read the corresponding byte mask file + open(unit=i+10,file=strmask,access='stream') + read(i+10)mask + CLOSE(i+10) + masks(:,:,i)=mask + + ! Free up memory with deallocation + deallocate(dat) + deallocate(mask) + + ! compute the reference phase for ith interferogram (by taking average of all reference pixels) + phase_ref(i)=0. + counts = 0 + do k=1,n_refs + if(masks(ref_locs(1,k),ref_locs(2,k),i).ne.0)then + phase_ref(i)=phase_ref(i)+phase(ref_locs(1,k),ref_locs(2,k),i) + counts=counts+1 + end if + end do + + ! Calculate the number of valid reference pixels available and either divide the summed reference phase by that value or, + ! if there were no valid pixels, take the average of the valid pixels as the reference phase. + if (counts /= 0) then + phase_ref(i)=phase_ref(i)/real(counts) + else + do k=1,nr + do kk=1,naz + if (masks(k,kk,i) /= 0) then + phase_ref(i)=phase_ref(i) + phase(k,kk,i) + counts=counts+1 + end if + end do + end do + if (counts /= 0) then + phase_ref(i)=phase_ref(i)/real(counts) + end if + end if + ! if count still equals zero, then there are no valid pixels in this interferogram, and phase_ref(i) can be left 0 + + ! subtract reference phase from each interferogram + phase(:,:,i)=phase(:,:,i)-phase_ref(i) + + END DO + !$OMP end parallel do + + ! adjust the amplitudes that were read in, since they were in powers, not amplitude. + amps=sqrt(amps) + + PRINT*,'Data Loaded' + + !!!!!! SBAS LEAST SQUARES AND STACKING !!!!!!! + ! at each pixel, solve for velocity at (nslc-1) time interval + ! uses internal subroutine pinv (which relies on external fortran function svd.f90) + ! uses built-in fortran intrinsic function matmul + + ! Allocate velocity solution matrix, stack, stack parameters, and first/last indices + ALLOCATE(velocity(nr,naz,nslc-1)) + ALLOCATE(amp(nr,naz),stacktime(nr,naz),stack(nr,naz),npts(nr,naz),idxf(nr,naz),idxl(nr,naz)) + + velocity(:,:,:)=-99999. + amp(:,:)=0. + stacktime=0. + stack=0. + npts = 0 + idxf = -99999 + idxl = -99999 + + + do i=1,nunwlists + + PRINT*,'Working on ',trim(adjustl(unwnames(i))) + + ! Read in the unwlist from unwnames(i) using subroutine read_list + call read_list(trim(adjustl(unwnames(i))), unwlist, nunw_temp) + + PRINT*,' ','unwlist read: ',nunw_temp + if (nunw_temp == 0) then + PRINT*,' ','No Unwrapped File in Group, skipping...' + cycle + end if + ! Read in the corresponding geolist to get number of valid slcs using subroutine read_list + geolist_name = Replace_Text(trim(adjustl(unwnames(i))),'unwlist','geolist') + call read_list(geolist_name, geolist_temp, nvalid) + + PRINT*,' ','geolist read: ',nvalid + + ! Read in the pixel list similar to reading in the reference pixel file (I think) + pixel_file = Replace_Text(trim(adjustl(unwnames(i))),'unwlist','pixels') + PRINT*,' ',pixel_file + + open(unit=66,file=pixel_file,status='OLD',action='READ') + npix=0 + DO + READ(66,'(A)',IOSTAT=ios)str + if (ios /= 0) EXIT + npix = npix+1 + END DO + + ALLOCATE(pixels(2,npix)) + REWIND 66 + + do k=1,npix + read(66,*,IOSTAT=ios)pixels(1,k),pixels(2,k) + IF (ios /= 0) EXIT + end do + close(66) + + PRINT*,' ','Pixel list read in: ',npix + + ! Create an array from 'Tm.out' (currently hard-coded) + ALLOCATE(Tm(nunw_temp,nvalid-1)) + Tmstr = Replace_text(trim(adjustl(unwnames(i))),'unwlist','Tm') + Tmstr = TRIM(Tmstr) // '.out' + PRINT*,' ',Tmstr + OPEN(24,FILE=Tmstr,STATUS='old') + do k=1,nunw_temp + READ(24,*,IOSTAT=stat)(Tm(k,kk),kk=1,nvalid-1) + end do + CLOSE(24) + + PRINT*,' ','Tm.out file read in' + + ! Read in the appropriate timedeltas array + ALLOCATE(timedeltas_temp(nvalid-1)) + timedeltas_file = Replace_Text(Tmstr,'Tm','timedeltas') + open(29,file=timedeltas_file,status='old') + read(29,*)timedeltas_temp + close(29) + + PRINT*,' ','timedeltas file read in' + + !!!!! Allocate and calculate other arrays unique to the pixel group + + ! Calculate Tminv with internal subroutine pinv + ALLOCATE(Tminv(nvalid-1,nunw_temp)) + call pinv(Tm,nunw_temp,nvalid-1,Tminv) + DEALLOCATE(Tm) + + PRINT*,' Tm inverted to Tminv' + + ! Make a cumulative sum of the timedeltas array in order to allocate the sub-velocity solution to the full + ALLOCATE(cumsumt(nvalid-1)) + do k=1,nvalid-1 + cumsumt(k) = sum(timedeltas_temp(1:k)) + end do + + DEALLOCATE(timedeltas_temp) + PRINT*,' cumulative sum of timdeltas calculated' + + ! Date of first SLC acquisition + date2 = trim(adjustl(geolist_temp(1))) + READ(date2(1:4),*)year2 + READ(date2(5:6),*)month2 + READ(date2(7:8),*)day2 + + ! Convert dates to days since a reference date (e.g., 1 January 1900) + days2 = date_to_days(year2, month2, day2) + + ! Calculate the difference + days_between = days2 - days1 + + ! Adjust the cumsumt array to account for the time difference between the true first acquisition and this one + cumsumt = cumsumt + days_between + + ! Find the indices of the first and last valid dates, to avoid extrapolation + do k=1,nslc + if (geolist (k) == geolist_temp(1)) then + do kk=1,npix + idxf(pixels(1,kk),pixels(2,kk)) = k-1 + end do + end if + if (geolist (k) == geolist_temp(nvalid)) then + do kk=1,npix + idxl(pixels(1,kk),pixels(2,kk)) = k-1 + end do + end if + end do + + ! Find the index of the unwrapped interferogram that is in the unwlist and extract phase and amplitude + ALLOCATE(phase_temp(npix,nunw_temp),amp_temp(npix,nunw_temp)) + ALLOCATE(deltime_temp(nunw_temp)) + + !$OMP parallel do private(kk,idx) shared(cells,unwlist,deltime,deltime_temp,phase,amps,pixels,phase_temp,amp_temp) + do k=1,nunw_temp + idx=-1 + do kk=1,ncells + if (trim(adjustl(cells(kk))) == trim(adjustl(unwlist(k)))) then + idx = kk + exit + end if + end do + + if (idx /= -1) then + deltime_temp(k) = deltime(2,idx) + do kk=1,npix + phase_temp(kk,k) = phase(pixels(1,kk),pixels(2,kk),idx) + amp_temp(kk,k) = amps(pixels(1,kk),pixels(2,kk),idx) + end do + end if + end do + !$OMP end parallel do + PRINT*,' Phase and amplitude at each pixel in group extracted' + + ! Calculate the number of days across all valid interferograms + ndays_temp = sum(deltime_temp) + + ! Run SBAS on each pixel and calculate stacks & stack parameters + ALLOCATE(vel_temp(npix,nvalid-1)) + vel_temp=0 + + ALLOCATE(temp1(nunw_temp)) + + PRINT*,' Calculating Velocity and Stack' + !$OMP parallel do private(temp1) shared(npix,phase_temp,pixels,vel_temp,nunw_temp,ndays_temp,Tminv,npts,stacktime,amp,stack) + do kk=1,npix + temp1=phase_temp(kk,:) + temp1=temp1(:) + vel_temp(kk,:)=MATMUL(Tminv,temp1) + + npts(pixels(1,kk),pixels(2,kk)) = nunw_temp + stacktime(pixels(1,kk),pixels(2,kk)) = ndays_temp + + if (nunw_temp /= 0) then + amp(pixels(1,kk),pixels(2,kk)) = sum(amp_temp(kk,:))/nunw_temp + end if + + if (ndays_temp /= 0) then + stack(pixels(1,kk),pixels(2,kk)) = sum(phase_temp(kk,:))/ndays_temp + end if + + end do + !$OMP end parallel do + + DEALLOCATE(temp1) + DEALLOCATE(Tminv) + DEALLOCATE(phase_temp) + DEALLOCATE(amp_temp) + DEALLOCATE(deltime_temp) + + + ! Fill in the appropriate velocity values in the full solution + PRINT*,' Interpolating velocity solution to full time series' + if (idxf(pixels(1,1),pixels(2,1)) /= -99999)then + tempday = 0 + if (idxf(pixels(1,1),pixels(2,1))>0) then + tempday = cumsumt(idxf(pixels(1,1),pixels(2,1))) + end if + do k=1,nvalid-1 + do j = k,nslc-1 + if (cumsumf(j).le.cumsumt(k).and.cumsumf(j)>tempday)then + do kk=1,npix + velocity(pixels(1,kk),pixels(2,kk),j) = vel_temp(kk,k) + end do + end if + if (cumsumf(j)==cumsumt(k))then + exit + end if + end do + tempday=cumsumt(k) + end do + end if + + DEALLOCATE(vel_temp) + DEALLOCATE(pixels) + DEALLOCATE(cumsumt) + + end do + + !!!!! WRITE RESULTS TO FILES !!!!! + print *,'Writing velocity and displacement solutions' + OPEN(28,FILE='velocity',FORM='unformatted',ACCESS='stream') + WRITE(28)velocity + close(28) + + OPEN(29,FILE='displacement',FORM='unformatted',ACCESS='direct',RECL=nr*8) + + ! integrate velocities for displacement with parallelization of outer loop (each time step) + !$OMP parallel do private(j,k,disp) shared(nslc,naz,nr,velocity,timedeltas,amp) + do i=1,nslc-1 + do j=1,naz + disp=-99999 + do k=1,nr + if (idxf(k,j) /= -99999)then + if (i.le.idxl(k,j).and.idxf(k,j)+1.le.i) then + disp(k)=sum(velocity(k,j,idxf(k,j)+1:i)*timedeltas(idxf(k,j)+1:i)) + end if + end if + end do + + ! write out average amplitude and displacement for each azimuth line and time step pair + write(29,rec=j+(i-1)*naz)amp(:,j),disp + end do + end do + !$OMP end parallel do + close(29) + + !!!!!! write stack files for further reference !!!!!! + print *,'Writing stacks' + ! Write out file for the number of valid unwrapped files for each pixel + OPEN(28,FILE='npts',FORM='unformatted',ACCESS='stream') + WRITE(28)npts + close(28) + + ! Write out file for the total number of days spanned by the valid interferograms for each pixel + OPEN(28,FILE='stacktime',FORM='unformatted',ACCESS='stream') + WRITE(28)stacktime + close(28) + + ! Write out the file describing the indices of the first valid date at each pixel. 0=t0 + OPEN(28,FILE='index_first',FORM='unformatted',ACCESS='stream') + WRITE(28)idxf + close(28) + + ! Write out the file describing the indices of the last valid date at each pixel. 0=t0 + OPEN(28,FILE='index_last',FORM='unformatted',ACCESS='stream') + WRITE(28)idxl + close(28) + + + ! Write weighted amps and phases stack into stackmht, phases in rad/day (pixel-interleaved format?) + OPEN(28,FILE='stackmht',FORM='unformatted',ACCESS='direct',RECL=nr*8) + do i=1,naz + WRITE(28,rec=i)amp(:,i),stack(:,i) + end do + CLOSE(28) + + ! Save nr, naz, nslc, ncells parameters in one file + OPEN(28,FILE='parameters',STATUS='replace') + WRITE(28,*) nr, naz, nslc, ncells + CLOSE(28) + + + PRINT*,'Data written' + + + !!!!! FUNCTIONS !!!!! + + CONTAINS + + ! ------------------ Replace_Text ----------------------- ! + FUNCTION Replace_Text (s,text,rep) RESULT(outs) + CHARACTER(*) :: s,text,rep + CHARACTER(LEN(s)+100) :: outs ! provide outs with extra 100 char len + INTEGER :: i, nt, nr + + outs = s ; nt = LEN_TRIM(text) ; nr = LEN_TRIM(rep) + DO + i = INDEX(outs,text(:nt)) ; IF (i == 0) EXIT + outs = outs(:i-1) // rep(:nr) // outs(i+nt:) + END DO + END FUNCTION Replace_Text + + + ! ----------------------- pinv --------------------------- ! + subroutine pinv(mat,rows,cols,matinv) + IMPLICIT none + + !specifications + INTEGER,INTENT(IN)::rows,cols + INTEGER :: ierr,kk,r,c + REAL,DIMENSION(rows,cols),INTENT(IN)::mat + REAL,DIMENSION(cols,rows)::matinv + REAL*8,DIMENSION(:),allocatable::Sing + REAL*8,DIMENSION(:,:),allocatable::Sinv,dmat,U,V,X,S + + allocate (dmat(rows,cols),Sinv(cols,cols)) + allocate (U(rows,cols),V(cols,cols),Sing(cols),S(cols,cols)) + + !executions + do r=1,rows + do c=1,cols + dmat(r,c)=mat(r,c) + !print *,r,c,mat(r,c),dmat(r,c) + end do + end do + + ! Call routine svd, which was compiled with the sbas_multi.f90 function + call svd(rows,cols,dmat,Sing,.true.,U,.true.,V,ierr) + + S=0. + do c=1,cols + S(c,c)=Sing(c) + end do + + Sinv=0. + DO r=1,cols + if(abs(S(r,r)).gt.1.e-6)Sinv(r,r)=1./S(r,r) + END DO + + ! Not sure where the sngl is coming from + matinv(:,:)=sngl(MATMUL(MATMUL(V,Sinv),TRANSPOSE(U))) + + deallocate (dmat) + deallocate (V) + deallocate (Sing) + deallocate (Sinv) + deallocate (U) + END subroutine pinv + + + ! -------------------------- read_list ----------------------------- ! + subroutine read_list(filename, string_array, num_lines) + implicit none + + ! Input arguments + character(len=*), intent(in) :: filename + ! Output arguments + character(len=*), dimension(:), allocatable, intent(out) :: string_array + integer, intent(out) :: num_lines + + integer :: i, ios, max_lines + character(len=100) :: line + integer, parameter :: default_max_lines = 10000 + + ! Temporary variables + integer :: line_count + character(len=100), dimension(default_max_lines) :: temp_array + + ! Open the file for reading + open(unit=23, file=filename, status='old', action='read', iostat=ios) + if (ios /= 0) then + print *, 'Error opening file: ', filename + stop + end if + + ! Initialize line counter + line_count = 0 + + ! Count lines + do + read(23, '(A)', iostat=ios) line + if (ios /= 0) exit + line_count = line_count + 1 + end do + + ! Close the file after counting + close(23) + + ! Allocate array based on the number of lines + allocate(string_array(line_count)) + + ! Reopen the file for reading strings into array + open(unit=23, file=filename, status='old', action='read', iostat=ios) + if (ios /= 0) then + print *, 'Error reopening file: ', filename + stop + end if + + ! Read lines into the array + do i = 1, line_count + read(23, '(A)', iostat=ios) string_array(i) + if (ios /= 0) then + print *, 'Error reading line ', i + stop + end if + end do + + ! Close the file after reading + close(23) + + ! Set the output argument + num_lines = line_count + + end subroutine read_list + + ! ------------------ date_to_days ------------------------- ! + ! Function to convert a date to days since 1 January 1900 + function date_to_days(year, month, day) result(days) + integer, intent(in) :: year, month, day + integer :: days, i + + ! Reference date: 1 January 1900 + integer, parameter :: ref_year = 1900 + integer, parameter :: ref_month = 1 + integer, parameter :: ref_day = 1 + + ! Days from reference date + days = 0 + + ! Add days for full years + do i = ref_year, year - 1 + days = days + 365 + if (mod(i, 4) == 0 .and. (mod(i, 100) /= 0 .or. mod(i, 400) == 0)) then + days = days + 1 + end if + end do + + ! Add days for full months of current year + do i = 1, month - 1 + days = days + days_in_month(year, i) + end do + + ! Add days for current month + days = days + day - ref_day + + end function date_to_days + + ! ----------------------- days_in_month ------------------- ! + ! Function to return the number of days in a given month of a given year + function days_in_month(year, month) result(days) + integer, intent(in) :: year, month + integer :: days + + select case (month) + case (1, 3, 5, 7, 8, 10, 12) + days = 31 + case (4, 6, 9, 11) + days = 30 + case (2) + if (mod(year, 4) == 0 .and. (mod(year, 100) /= 0 .or. mod(year, 400) == 0)) then + days = 29 + else + days = 28 + end if + case default + days = 0 + end select + + end function days_in_month + +END PROGRAM sbas_multi diff --git a/kp_scripts/sbas/sbas_setup.py b/kp_scripts/sbas/sbas_setup.py new file mode 100644 index 0000000..dd6fc93 --- /dev/null +++ b/kp_scripts/sbas/sbas_setup.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 + +# sbas_setup.py - create auxiliary files for Ann's sbas matlab code specific to a pixel group's geolist/sbas list + +import sys +from datetime import datetime +import os +import math + +if len(sys.argv) < 3: + print ('Usage: sbas_setup.py sbas_list geolist ') + sys.exit(1) + +sbas_list=sys.argv[1] +peg_list=sys.argv[2] +suff = '' +if len(sys.argv) == 4: + suff=sys.argv[3] + +fpeg=open(peg_list,'r') +peg=fpeg.readlines() + +slctime=list(float(i) for i in range(0, len(peg))) +deltatime=list(float(i) for i in range(0, len(peg))) +k=0 +for line in peg: + words=line.split() + slc=words[-1] + first=slc.find('20') + slcname=slc[first:first+8] + slcyear=slcname[0:4] + slcmon=slcname[4:6] + slcday=slcname[6:8] + jd=datetime.strptime(slcname, '%Y%m%d').toordinal()+1721424.5 + slctime[k] = jd #int(slcyear)+float(int(slcmon))/12.+float(int(slcday))/360. + if k > 0: + deltatime[k] = slctime[k]-slctime[k-1] + k=k+1 + + +timedel = 'timedeltas'+suff+'.out' +ftimedeltas=open(timedel,'w') # save time intervals between slcs +for i in range(1,len(deltatime)): + ftimedeltas.write(str(deltatime[i])+'\n') + +fsbas=open(sbas_list,'r') +sbas=fsbas.readlines() + +fbperp=open('Bperp'+suff+'.out','w') +fdeltime=open('deltime'+suff+'.out','w') +ftm=open('Tm'+suff+'.out','w') + +id=0 +for line in sbas: + words=line.split() + primary=words[0] + secondary=words[1] + timebaseline=words[2] + spacebaseline=words[3] + +# get a short name for primary and secondary files + first=primary.find('20') + primaryname=primary[first:first+8] + primaryyear=primaryname[0:4] + primarymon=primaryname[4:6] + primaryday=primaryname[6:8] + slc1time=datetime.strptime(primaryname, '%Y%m%d').toordinal()+1721424.5 + + first=secondary.find('20') + secondaryname=secondary[first:first+8] + secondaryyear=secondaryname[0:4] + secondarymon=secondaryname[4:6] + secondaryday=secondaryname[6:8] + slc2time=datetime.strptime(secondaryname, '%Y%m%d').toordinal()+1721424.5 + + # spatial baseline to Bperp.out + fbperp.write(spacebaseline+'\n') + + # temporal baseline to deltime.out + id=id+1 + fdeltime.write(str(id)+' '+timebaseline+' '+str(slc1time)+' '+str(slc2time)+'\n') + + # time interval matrix to Tm.out + # which slc for primary and secondary? + k=0 + for line in peg: + words=line.split() + slc=words[-1] + first=slc.find('20') + slcname=slc[first:first+8] + if slcname == primaryname: + kprimary=k + if slcname == secondaryname: + ksecondary=k + k=k+1 + + times=list(float(0) for i in range (0,len(peg))) +# print kprimary,' ',ksecondary + if kprimary < ksecondary: + for i in range(kprimary,ksecondary): + times[i]=deltatime[i+1] + if kprimary > ksecondary: + for i in range(ksecondary,kprimary): + times[i]=-deltatime[i+1] + for i in range(0,len(times)-1): + ftm.write(str(times[i])+' ') + ftm.write('\n') + +sys.exit() + diff --git a/kp_scripts/sbas/sbas_unique b/kp_scripts/sbas/sbas_unique new file mode 100644 index 0000000..88ab17c Binary files /dev/null and b/kp_scripts/sbas/sbas_unique differ diff --git a/kp_scripts/sbas/sbas_unique.f90 b/kp_scripts/sbas/sbas_unique.f90 new file mode 100644 index 0000000..d2f82d3 --- /dev/null +++ b/kp_scripts/sbas/sbas_unique.f90 @@ -0,0 +1,732 @@ +! Run SBAS on unique pixel groups +! Each pixel has a vector describing whether or not that interferogram should be +! included in the analysis. +! Optional: load in an array describing the maximum temporal baseline to use at +! each pixel. +! Incomplete velocity solutions are interpolated prior to integration to +! displacement. No velocity extrapolation (filled with NaN values, or -99999) + + +! Compile with: +! gfortran -o sbas_unique sbas_unique.f90 svd.f90 -fopenmp -lrt -lpthread + +PROGRAM sbas_unique + use omp_lib + + IMPLICIT none + + !!!!! SPECIFICATIONS !!!!! + + ! INPUTS + INTEGER:: ncells,nslc + INTEGER*8::nr + CHARACTER(100)::filelist,str,Tmfile,timedeltafile,deltimefile,reflocs_file,scenemaskfile,maxTBfile + CHARACTER(4)::intmaskflag + + ! ERROR/STATUS CHECKS/COUNTING VARIABLES + INTEGER::i,k,kk,counts,j,stat,ierr,stat2,ios + INTEGER,DIMENSION(13)::statb + + ! READING IN INPUTS LISTS AND DEFINING OTHER VARIABLES + INTEGER::n_refs + INTEGER*8::naz,x,y + INTEGER,DIMENSION(:,:),ALLOCATABLE::ref_locs + CHARACTER(100),DIMENSION(:),ALLOCATABLE::cells + + ! DATA ARRAYS FOR SET OF ALL INTERFEROGRAMS + INTEGER*1,DIMENSION(:,:,:),ALLOCATABLE::masks + INTEGER*1,DIMENSION(:,:),ALLOCATABLE::scenemask + REAL,DIMENSION(:,:,:),ALLOCATABLE::phase,amps + INTEGER,DIMENSION(:,:),ALLOCATABLE::maxTB + + ! ARRAYS FOR SUMMARIZING/REDUCING FULL DATA SET + INTEGER,DIMENSION(:,:),ALLOCATABLE::npts,idxf,idxl + REAL,DIMENSION(:),ALLOCATABLE::phase_ref,disp + REAL,DIMENSION(:,:),ALLOCATABLE::amp,stacktime,stack + + ! FOR LOADING FULL TIME SERIES INFORMATION (ARRAYS) + REAL,DIMENSION(:),ALLOCATABLE::timedeltas + REAL,DIMENSION(:,:),ALLOCATABLE::deltime + + ! LOADING IN THE UNWRAPPED INTERFEROGRAMS, AMPLITUDES, AND MASKS; FIND REFERENCE PHASE AT EACH INTERFEROGRAM + INTEGER*1,DIMENSION(:,:),ALLOCATABLE::mask + CHARACTER(100)::strint,stramp,strmask + REAL,DIMENSION(:,:),ALLOCATABLE::dat + + ! FULL SOLUTION + REAL,DIMENSION(:,:,:),ALLOCATABLE::velocity,velocity_interp + + ! PIXEL-WISE SBAS + REAL,DIMENSION(:),ALLOCATABLE::v_in,v_out,phase_temp,amp_temp,deltime_temp + REAL,DIMENSION(:,:),ALLOCATABLE::Tm,Tminv,Tm_temp,Tminv_temp + INTEGER,DIMENSION(:),ALLOCATABLE::mask_vector + INTEGER::nrows,ndays,idx + + + + !!!!! EXECUTIONS !!!!! + ! check to see if proper number of arguments were provided + IF(iargc().lt.7)THEN + WRITE(*,*)'usage: sbas_unique intlist nints nslc nr Tmfile timedeltafile' // & + ' deltimefile ' // & + ' ' + STOP + END IF + + ! read in arguments and convert to appropriate type (where necessary) + CALL getarg(1,filelist) + CALL getarg(2,str) + READ(str,*)ncells + CALL getarg(3,str) + READ(str,*)nslc + CALL getarg(4,str) + READ(str,*)nr + CALL getarg(5,Tmfile) + CALL getarg(6,timedeltafile) + CALL getarg(7,deltimefile) + + ! Set Default parameters and check for inputs + reflocs_file = 'none' + IF(iargc().ge.8)THEN + CALL getarg(8,reflocs_file) + END IF + + scenemaskfile = 'none' + IF(iargc().ge.9)THEN + CALL getarg(9,scenemaskfile) + END IF + + intmaskflag = 'N' + IF(iargc().ge.10)THEN + CALL getarg(10,intmaskflag) + END IF + + maxTBfile='none' + IF(iargc().ge.11)THEN + CALL getarg(11,maxTBfile) + END IF + + + ! allocate the list of unwrapped files and the reference pixel locations array + ALLOCATE(cells(ncells),ref_locs(2,10000000)) + + ! read in the list of unwrapped interferogram files + OPEN(UNIT=11,FILE=filelist,STATUS='old') + READ(11,'(A)',end=10,IOSTAT=stat)cells + 10 continue + CLOSE(11) + + ! Read in the first unwrapped interferogram name in order to determine the number of azimuth lines, given the known number of range lines + OPEN(UNIT=21,FILE=trim(adjustl(cells(1))),FORM='unformatted',ACCESS='direct',RECL=nr*8) + ierr=fstat(21,statb) + naz=statb(8)/8/nr + WRITE(*,*)' Lines in file: ',naz + CLOSE(21) + + + ! Set list of reference points, default to scene center if no input file is given + n_refs=1 + ref_locs(1,1)=nr/2 + ref_locs(2,1)=naz/2 + if(reflocs_file.ne.'none')then + open(unit=13,file=reflocs_file) + do i=1,10000000 + read(13,*,end=99)ref_locs(:,i) + end do + 99 n_refs=i-1 + CLOSE(13) + end if + print *,'Number of eligible reference points: ',n_refs + + + ! Read in scenemask if available + ALLOCATE(scenemask(nr,naz)) + scenemask=1 + IF(scenemaskfile.ne.'none')THEN + open(unit=13,file=scenemaskfile,access='stream') + read(13)scenemask + CLOSE(13) + END IF + + ! Read in the maxTBfile if available + ALLOCATE(maxTB(nr,naz)) + maxTB=10000 + IF(maxTBfile.ne.'none')THEN + open(unit=13,file=maxTBfile,access='stream') + read(13)maxTB + CLOSE(13) + END IF + + + + ! Now that we know some additional parameters, allocate some of the arrays we will need + ALLOCATE(phase(nr,naz,ncells),amps(nr,naz,ncells),masks(nr,naz,ncells)) + ALLOCATE(phase_ref(ncells)) + + ! Set values in arrays equal to zero for now + phase(:,:,:)=0. + phase_ref = 0. + amps=0. + masks(:,:,:)=0 + + + !!!!! READ IN THE SBAS PARAMETERS NEEDED FOR FULL TIME SERIES !!!!! + + ! Load in the model matrix Tm + ALLOCATE(Tm(ncells,nslc-1)) + OPEN(24,FILE=Tmfile,STATUS='old') + do k=1,ncells + READ(24,*,IOSTAT=stat)(Tm(k,kk),kk=1,nslc-1) + end do + CLOSE(24) + + + ! Create an array with data from deltime.out (currently hard-coded) + ALLOCATE(deltime(4,ncells)) + OPEN(13,FILE=deltimefile,STATUS='old') + DO i=1,ncells + READ(13,*,IOSTAT=stat)deltime(:,i) + END DO + CLOSE(13) + + ! Read time from slc to slc (i.e. temporal baseline between nearest neighbor scenes + ALLOCATE(timedeltas(nslc-1)) + open(13,file=timedeltafile,status='old') + read(13,*)timedeltas + close(13) + + + + ! read in unwrapped igrams using parallel computing and calculate the + ! reference phase for each interferogram, utilizing masks and maxTB, if + ! available (declared in inputs) + !$OMP parallel do private(k,kk,dat,strint,stramp,strmask,mask,counts,x,y) & + !$OMP shared(cells,ref_locs,deltime,nr,naz,ncells,phase,amps,phase_ref,masks,scenemask,maxTB) + DO i=1,ncells + ! allocate dat, which appears to be a “temporary” variable holding the unwrapped data + allocate(dat(nr*2,naz)) + IF(intmaskflag=='Y')THEN + allocate(mask(nr,naz)) + END IF + + ! get the correct names corresponding to the files that will be read in + strint=trim(adjustl(cells(i))) + stramp=Replace_Text(strint,'.unw','.amp') + + ! open the interferogram file directly. Requires conversion for reading. + OPEN(UNIT=i+10,FILE=strint,FORM='unformatted',ACCESS='direct',RECL=2*nr*naz*4,CONVERT= 'LITTLE_ENDIAN') + READ(i+10,rec=1)dat + CLOSE(i+10) + ! read the correct values from dat to get phase, and put in phase array + phase(:,:,i)=dat((nr+1):2*nr,:) + + ! Read the corresponding amplitude file, same way as the unwrapped file + OPEN(UNIT=i+10,FILE=stramp,FORM='unformatted',ACCESS='direct',RECL=2*nr*naz*4,CONVERT= 'LITTLE_ENDIAN') + READ(i+10,rec=1)dat + CLOSE(i+10) + ! read the correct values from dat to get the amplitude powers, put in amps array + amps(:,:,i)=dat(1:2*nr-1:2,:)**2+dat(2:2*nr:2,:)**2 + + IF(intmaskflag=='Y')THEN + strmask=Replace_Text(strint,'.unw','.mask') + ! read the corresponding byte mask file + open(unit=i+10,file=strmask,access='stream') + read(i+10)mask + CLOSE(i+10) + masks(:,:,i)=mask + deallocate(mask) + END IF + + ! Free up memory with deallocation + deallocate(dat) + + ! compute the reference phase for ith interferogram (by taking average of valid reference pixels) + phase_ref(i)=0. + counts = 0 + do k=1,n_refs + x = ref_locs(1,k) + y = ref_locs(2,k) + IF(deltime(i,2).le.maxTB(x,y))THEN + IF(intmaskflag=='Y')THEN + if(masks(x,y,i).ne.0)then + phase_ref(i)=phase_ref(i)+phase(x,y,i) + counts=counts+1 + end if + ELSE IF(scenemaskfile.ne.'none')THEN + if(scenemask(x,y).ne.0)then + phase_ref(i)=phase_ref(i)+phase(x,y,i) + counts=counts+1 + end if + ELSE + phase_ref(i)=phase_ref(i)+phase(x,y,i) + counts=counts+1 + END IF + END IF + end do + + ! Calculate the number of valid reference pixels available for the interferogram and either divide the summed reference phase by that value or, + ! if there were no valid pixels, take the average of the valid pixels as the reference phase. + if (counts /= 0) then + phase_ref(i)=phase_ref(i)/real(counts) + else + do k=1,nr + do kk=1,naz + IF (deltime(i,2).le.maxTB(k,kk)) THEN + IF (intmaskflag=='Y')THEN + if (masks(k,kk,i).ne.0) then + phase_ref(i)=phase_ref(i) + phase(k,kk,i) + counts=counts+1 + end if + ELSE IF (scenemaskfile.ne.'none') THEN + if (scenemask(k,kk).ne.0)then + phase_ref(i)=phase_ref(i) + phase(k,kk,i) + counts=counts+1 + end if + ELSE + phase_ref(i)=phase_ref(i) + phase(k,kk,i) + counts=counts+1 + END IF + END IF + end do + end do + if (counts /= 0) then + phase_ref(i)=phase_ref(i)/real(counts) + end if + end if + ! if count still equals zero, then there are no valid pixels in this interferogram, and phase_ref(i) can be left 0 + + ! subtract reference phase from each interferogram + phase(:,:,i)=phase(:,:,i)-phase_ref(i) + + END DO + !$OMP end parallel do + + ! adjust the amplitudes that were read in, since they were in powers, not amplitude. + amps=sqrt(amps) + + PRINT*,'Data Loaded' + + + + !!!!!! SBAS LEAST SQUARES AND STACKING !!!!!!! + ! at each pixel, solve for velocity at (nslc-1) time interval + ! uses internal subroutine pinv (which relies on external fortran function svd.f90) + ! uses built-in fortran intrinsic function matmul + + ! Allocate velocity solution matrix, stack, stack parameters, and first/last indices + ALLOCATE(velocity(nr,naz,nslc-1),velocity_interp(nr,naz,nslc-1),Tminv(nslc-1,ncells)) + ALLOCATE(amp(nr,naz),stacktime(nr,naz),stack(nr,naz),npts(nr,naz),idxf(nr,naz),idxl(nr,naz)) + + velocity(:,:,:)=0.0 + velocity_interp(:,:,:) = 0.0 + amp(:,:)=0. + stacktime=0. + stack=-99999. + npts = 0 + idxf = -99999 + idxl = -99999 + + ! Calculate the full Tminv so that you don't need to calculate it every + ! pixel unless you need to + call pinv(Tm,ncells,nslc-1,Tminv) + + + ! Run through each pixel, find the phase set that should be used, cull the + ! rows from the phase vector and Tm, invert for Tminv, and then solve for + ! preliminary velocity solution via matrix multiplication + ALLOCATE(mask_vector(ncells)) + ALLOCATE(v_in(nslc-1),v_out(nslc-1)) + + !$OMP parallel do collapse(2) private(mask_vector,nrows,phase_temp,amp_temp,deltime_temp) & + !$OMP private(Tm_temp,idx,ndays,Tminv_temp,k,v_in,v_out) & + !$OMP shared(scenemask,masks,maxTB,phase,Tm,amps,deltime,npts,stacktime,amp,stack) & + !$OMP shared(velocity,velocity_interp,idxf,idxl,timedeltas) + DO i=1,nr + DO j=1,naz + if(scenemask(i,j)==0)THEN + cycle + end if + + ! Find the appropriate mask vector so we can determine which rows to + ! remove + mask_vector = 1 + IF(intmaskflag=='Y')THEN + mask_vector = masks(i,j,:) + END IF + IF(maxTBfile.ne.'none')THEN + DO k=1,ncells + if(deltime(k,2).gt.maxTB(i,j))THEN + mask_vector(k)=0 + end if + END DO + END IF + + + ! create the temporary phase vector and Tm matrix + nrows = sum(real(mask_vector)) + if (nrows==0)THEN + cycle + end if + + + ALLOCATE(phase_temp(nrows),amp_temp(nrows),deltime_temp(nrows)) + if(nrows.ne.ncells)then + idx = 1 + ALLOCATE(Tm_temp(nrows,nslc-1)) + DO k=1,ncells + if(mask_vector(k).ne.0)THEN + phase_temp(idx) = phase(i,j,k) + Tm_temp(idx,:) = Tm(k,:) + amp_temp(idx) = amps(i,j,k) + deltime_temp(idx) = deltime(k,2) + idx=idx+1 + end if + END DO + else + phase_temp = phase(i,j,:) + amp_temp = amps(i,j,:) + deltime_temp = deltime(:,2) + end if + + ! Calculate the number of days across all valid interferograms + ndays = sum(deltime_temp) + npts(i,j) = nrows + stacktime(i,j) = ndays + + if (nrows /= 0) then + amp(i,j) = sum(amp_temp)/real(nrows) + stack(i,j) = sum(phase_temp)/real(ndays) + end if + + + if(nrows.ne.ncells)then + ! Get the temporary Tminv matrix + ! Calculate Tminv with internal subroutine pinv + ALLOCATE(Tminv_temp(nslc-1,nrows)) + call pinv(Tm_temp,nrows,nslc-1,Tminv_temp) + DEALLOCATE(Tm_temp) + + + ! Get the preliminary velocity solution via matrix multiplication + velocity(i,j,:)=MATMUL(Tminv_temp,phase_temp(:)) + + ! Get the first and last valid date for the pixel + DO k=1,nslc-1 + if (velocity(i,j,k).ne.0)THEN + idxf(i,j) = k + exit + end if + END DO + DO k=1,nslc-1 + if (velocity(i,j,nslc-k).ne.0)THEN + idxl(i,j) = nslc-k + exit + end if + END DO + deallocate(Tminv_temp) + else + velocity(i,j,:)=MATMUL(Tminv,phase_temp(:)) + idxf(i,j)=1 + idxl(i,j)=nslc-1 + end if + + ! Deallocate to save memory + DEALLOCATE(phase_temp,amp_temp,deltime_temp) + + + ! If there has been masking, check for any velocity values that are + ! identically 0. If they are, interpolate between valid velocity + ! estimates. No extrapolation though + if(nrows.ne.ncells)then + v_in = velocity(i,j,:) + v_in = v_in(:) + CALL interpolate_velocity(v_in,timedeltas,v_out) + velocity_interp(i,j,:) = v_out + else + velocity_interp(i,j,:) = velocity(i,j,:) + end if + + END DO + END DO + + + + + + !!!!! WRITE RESULTS TO FILES !!!!! + print *,'Writing velocity and displacement solutions' + OPEN(28,FILE='velocity',FORM='unformatted',ACCESS='stream') + WRITE(28)velocity + close(28) + + if (intmaskflag=='Y'.or.maxTBfile.ne.'none') then + OPEN(28,FILE='velocity_interp',FORM='unformatted',ACCESS='stream') + WRITE(28)velocity_interp + close(28) + end if + + OPEN(29,FILE='displacement',FORM='unformatted',ACCESS='direct',RECL=nr*8) + ALLOCATE(disp(nr)) + ! integrate velocities for displacement with parallelization of outer loop (each time step) + !$OMP parallel do private(j,k,disp) shared(velocity,timedeltas,amp) + do i=1,nslc-1 + do j=1,naz + do k=1,nr + disp(k) = sum(velocity(k,j,1:i)*timedeltas(1:i)) + end do + + ! write out average amplitude and displacement for each azimuth line and time step pair + write(29,rec=j+(i-1)*naz)amp(:,j),disp + end do + end do + !$OMP end parallel do + close(29) + + if (intmaskflag=='Y'.or.maxTBfile.ne.'none') then + OPEN(29,FILE='displacement_interp',FORM='unformatted',ACCESS='direct',RECL=nr*8) + !$OMP parallel do private(j,k,disp) shared(velocity,timedeltas,amp) + do i=1,nslc-1 + do j=1,naz + do k=1,nr + disp(k) = sum(velocity_interp(k,j,1:i)*timedeltas(1:i)) + end do + + ! write out average amplitude and displacement for each azimuth line + ! and time step pair + write(29,rec=j+(i-1)*naz)amp(:,j),disp + end do + end do + !$OMP end parallel do + close(29) + + end if + + + !!!!!! write stack files for further reference !!!!!! + print *,'Writing stacks' + ! Write out file for the number of valid unwrapped files for each pixel + OPEN(28,FILE='npts',FORM='unformatted',ACCESS='stream') + WRITE(28)npts + close(28) + + ! Write out file for the total number of days spanned by the valid interferograms for each pixel + OPEN(28,FILE='stacktime',FORM='unformatted',ACCESS='stream') + WRITE(28)stacktime + close(28) + + ! Write out the file describing the indices of the first valid date at each pixel. 0=t0 + OPEN(28,FILE='index_first',FORM='unformatted',ACCESS='stream') + WRITE(28)idxf + close(28) + + ! Write out the file describing the indices of the last valid date at each pixel. 0=t0 + OPEN(28,FILE='index_last',FORM='unformatted',ACCESS='stream') + WRITE(28)idxl + close(28) + + + ! Write weighted amps and phases stack into stackmht, phases in rad/day (pixel-interleaved format?) + OPEN(28,FILE='stackmht',FORM='unformatted',ACCESS='direct',RECL=nr*8) + do i=1,naz + WRITE(28,rec=i)amp(:,i),stack(:,i) + end do + CLOSE(28) + + ! Save nr, naz, nslc, ncells parameters in one file + OPEN(28,FILE='parameters',STATUS='replace') + WRITE(28,*) nr, naz, nslc, ncells + CLOSE(28) + + + PRINT*,'Data written' + + + !!!!! FUNCTIONS !!!!! + + CONTAINS + + ! ------------------ Replace_Text ----------------------- ! + FUNCTION Replace_Text (s,text,rep) RESULT(outs) + CHARACTER(*) :: s,text,rep + CHARACTER(LEN(s)+100) :: outs ! provide outs with extra 100 char len + INTEGER :: i, nt, nr + + outs = s ; nt = LEN_TRIM(text) ; nr = LEN_TRIM(rep) + DO + i = INDEX(outs,text(:nt)) ; IF (i == 0) EXIT + outs = outs(:i-1) // rep(:nr) // outs(i+nt:) + END DO + END FUNCTION Replace_Text + + + ! ----------------------- pinv --------------------------- ! + subroutine pinv(mat,rows,cols,matinv) + IMPLICIT none + + !specifications + INTEGER,INTENT(IN)::rows,cols + INTEGER :: ierr,kk,r,c + REAL,DIMENSION(rows,cols),INTENT(IN)::mat + REAL,DIMENSION(cols,rows)::matinv + REAL*8,DIMENSION(:),allocatable::Sing + REAL*8,DIMENSION(:,:),allocatable::Sinv,dmat,U,V,X,S + + allocate (dmat(rows,cols),Sinv(cols,cols)) + allocate (U(rows,cols),V(cols,cols),Sing(cols),S(cols,cols)) + + !executions + do r=1,rows + do c=1,cols + dmat(r,c)=mat(r,c) + !print *,r,c,mat(r,c),dmat(r,c) + end do + end do + + ! Call routine svd, which was compiled with the sbas_multi.f90 function + call svd(rows,cols,dmat,Sing,.true.,U,.true.,V,ierr) + + S=0. + do c=1,cols + S(c,c)=Sing(c) + end do + + Sinv=0. + DO r=1,cols + if(abs(S(r,r)).gt.1.e-6)Sinv(r,r)=1./S(r,r) + END DO + + ! Not sure where the sngl is coming from + matinv(:,:)=sngl(MATMUL(MATMUL(V,Sinv),TRANSPOSE(U))) + + deallocate (dmat) + deallocate (V) + deallocate (Sing) + deallocate (Sinv) + deallocate (U) + END subroutine pinv + + + ! ---------------------- interpolate_velocity ---------------------- ! + subroutine interpolate_velocity(velocity, dt, velocity_interp) + implicit none + real, dimension(:), intent(in) :: velocity, dt + real, dimension(:), intent(out) :: velocity_interp + integer :: i, start_idx, end_idx,k,sday + real :: total_days, interp_value + real :: vleft,vright,nday,dright,dleft + real,dimension(:),allocatable :: v + + ! Copy the original velocity to the output (keeping non-zero values) + velocity_interp = velocity + + ! Loop over the array and find intervals with zeros (NaNs) + i = 1 + do while (i <= size(velocity)) + ! Skip if current element is non-zero + if (velocity(i) /= 0.0) then + i = i + 1 + cycle + end if + + ! Find the start and end of a zero (NaN) region + start_idx = i + do while (i <= size(velocity) .and. velocity(i) == 0.0) + i = i + 1 + end do + end_idx = i + + ! Ensure there are valid values on both sides + if (start_idx > 1 .and. end_idx <= size(velocity)) then + if (velocity(start_idx - 1) /= 0.0 .and. velocity(end_idx) /= 0.0)then + ! Compute the total days between the endpoints + total_days = sum(dt(start_idx:end_idx-1))+1 + + ! Interpolate each zero position based on its relative days + ! weight + vleft = velocity(start_idx-1) + vright = velocity(end_idx) + do i = start_idx, end_idx - 1 + nday = dt(i) + sday = int(1 + sum(dt(1:i)) - nday) + ALLOCATE(v(int(nday))) + DO k=sday,sday+int(nday) + dright = total_days-k + dleft = k + v(k) = ((dright*vleft)+(dleft*vright))/total_days + END DO + velocity_interp(i) = sum(v)/nday + DEALLOCATE(v) + end do + end if + end if + end do + end subroutine interpolate_velocity + + + ! -------------------------- read_list ----------------------------- ! + subroutine read_list(filename, string_array, num_lines) + implicit none + + ! Input arguments + character(len=*), intent(in) :: filename + ! Output arguments + character(len=*), dimension(:), allocatable, intent(out) :: string_array + integer, intent(out) :: num_lines + + integer :: i, ios, max_lines + character(len=100) :: line + integer, parameter :: default_max_lines = 10000 + + ! Temporary variables + integer :: line_count + character(len=100), dimension(default_max_lines) :: temp_array + + ! Open the file for reading + open(unit=23, file=filename, status='old', action='read', iostat=ios) + if (ios /= 0) then + print *, 'Error opening file: ', filename + stop + end if + + ! Initialize line counter + line_count = 0 + + ! Count lines + do + read(23, '(A)', iostat=ios) line + if (ios /= 0) exit + line_count = line_count + 1 + end do + + ! Close the file after counting + close(23) + + ! Allocate array based on the number of lines + allocate(string_array(line_count)) + + ! Reopen the file for reading strings into array + open(unit=23, file=filename, status='old', action='read', iostat=ios) + if (ios /= 0) then + print *, 'Error reopening file: ', filename + stop + end if + + ! Read lines into the array + do i = 1, line_count + read(23, '(A)', iostat=ios) string_array(i) + if (ios /= 0) then + print *, 'Error reading line ', i + stop + end if + end do + + ! Close the file after reading + close(23) + + ! Set the output argument + num_lines = line_count + + end subroutine read_list + + +END PROGRAM sbas_unique diff --git a/kp_scripts/sbas/svd.f90 b/kp_scripts/sbas/svd.f90 new file mode 100644 index 0000000..55a4f12 --- /dev/null +++ b/kp_scripts/sbas/svd.f90 @@ -0,0 +1,557 @@ +subroutine svd ( m, n, a, w, matu, u, matv, v, ierr ) + +!*****************************************************************************80 +! +!! SVD computes the singular value decomposition for a real matrix. +! +! Discussion: +! +! This subroutine determines the singular value decomposition +! +! A = U * S * V' +! +! of a real M by N rectangular matrix. Householder bidiagonalization +! and a variant of the QR algorithm are used. +! +! Licensing: +! +! This code is distributed under the GNU LGPL license. +! +! Modified: +! +! 18 October 2009 +! +! Author: +! +! Original FORTRAN77 version by Smith, Boyle, Dongarra, Garbow, Ikebe, +! Klema, Moler. +! FORTRAN90 version by John Burkardt. +! +! Reference: +! +! Golub, Christian Reinsch, +! Numerische Mathematik, +! Volume 14, 1970, pages 403-420. +! +! James Wilkinson, Christian Reinsch, +! Handbook for Automatic Computation, +! Volume II, Linear Algebra, Part 2, +! Springer, 1971, +! ISBN: 0387054146, +! LC: QA251.W67. +! +! Brian Smith, James Boyle, Jack Dongarra, Burton Garbow, +! Yasuhiko Ikebe, Virginia Klema, Cleve Moler, +! Matrix Eigensystem Routines, EISPACK Guide, +! Lecture Notes in Computer Science, Volume 6, +! Springer Verlag, 1976, +! ISBN13: 978-3540075462, +! LC: QA193.M37. +! +! Parameters: +! +! Input, integer ( kind = 4 ) M, the number of rows of A and U. +! +! Input, integer ( kind = 4 ) N, the number of columns of A and U, and +! the order of V. +! +! Input, real ( kind = 8 ) A(M,N), the M by N matrix to be decomposed. +! +! Output, real ( kind = 8 ) W(N), the singular values of A. These are the +! diagonal elements of S. They are unordered. If an error exit is +! made, the singular values should be correct for indices +! IERR+1, IERR+2,..., N. +! +! Input, logical MATU, should be set to TRUE if the U matrix in the +! decomposition is desired, and to FALSE otherwise. +! +! Output, real ( kind = 8 ) U(M,N), contains the matrix U, with orthogonal +! columns, of the decomposition, if MATU has been set to TRUE. Otherwise +! U is used as a temporary array. U may coincide with A. +! If an error exit is made, the columns of U corresponding +! to indices of correct singular values should be correct. +! +! Input, logical MATV, should be set to TRUE if the V matrix in the +! decomposition is desired, and to FALSE otherwise. +! +! Output, real ( kind = 8 ) V(N,N), the orthogonal matrix V of the +! decomposition if MATV has been set to TRUE. Otherwise V is not referenced. +! V may also coincide with A if U is not needed. If an error +! exit is made, the columns of V corresponding to indices of +! correct singular values should be correct. +! +! Output, integer ( kind = 4 ) IERR, error flag. +! 0, for normal return, +! K, if the K-th singular value has not been determined after 30 iterations. +! + implicit none + + integer ( kind = 4 ) m + integer ( kind = 4 ) n + + real ( kind = 8 ) a(m,n) + real ( kind = 8 ) c + real ( kind = 8 ) f + real ( kind = 8 ) g + real ( kind = 8 ) h + integer ( kind = 4 ) i + integer ( kind = 4 ) ierr + integer ( kind = 4 ) its + integer ( kind = 4 ) i1 + integer ( kind = 4 ) j + integer ( kind = 4 ) k + integer ( kind = 4 ) kk + integer ( kind = 4 ) k1 + integer ( kind = 4 ) l + integer ( kind = 4 ) ll + integer ( kind = 4 ) l1 + logical matu + logical matv + integer ( kind = 4 ) mn + real ( kind = 8 ) pythag + real ( kind = 8 ) rv1(n) + real ( kind = 8 ) s + real ( kind = 8 ) scale + real ( kind = 8 ) tst1 + real ( kind = 8 ) tst2 + real ( kind = 8 ) u(m,n) + real ( kind = 8 ) v(n,n) + real ( kind = 8 ) w(n) + real ( kind = 8 ) x + real ( kind = 8 ) y + real ( kind = 8 ) z + + ierr = 0 + u(1:m,1:n) = a(1:m,1:n) +! +! Householder reduction to bidiagonal form. +! + g = 0.0D+00 + scale = 0.0D+00 + x = 0.0D+00 + + do i = 1, n + + l = i + 1 + rv1(i) = scale * g + g = 0.0D+00 + s = 0.0D+00 + scale = 0.0D+00 + + if ( i <= m ) then + + scale = sum ( abs ( u(i:m,i) ) ) + + if ( scale /= 0.0D+00 ) then + + u(i:m,i) = u(i:m,i) / scale + + s = sum ( u(i:m,i)**2 ) + + f = u(i,i) + g = - sign ( sqrt ( s ), f ) + h = f * g - s + u(i,i) = f - g + + if ( i /= n ) then + + do j = l, n + s = dot_product ( u(i:m,i), u(i:m,j) ) + u(i:m,j) = u(i:m,j) + s * u(i:m,i) / h + end do + + end if + + u(i:m,i) = scale * u(i:m,i) + + end if + + end if + + w(i) = scale * g + g = 0.0D+00 + s = 0.0D+00 + scale = 0.0D+00 + + if ( i <= m .and. i /= n ) then + + scale = sum ( abs ( u(i,l:n) ) ) + + if ( scale /= 0.0D+00 ) then + + u(i,l:n) = u(i,l:n) / scale + s = sum ( u(i,l:n)**2 ) + f = u(i,l) + g = - sign ( sqrt ( s ), f ) + h = f * g - s + u(i,l) = f - g + rv1(l:n) = u(i,l:n) / h + + if ( i /= m ) then + + do j = l, m + + s = dot_product ( u(j,l:n), u(i,l:n) ) + + u(j,l:n) = u(j,l:n) + s * rv1(l:n) + + end do + + end if + + u(i,l:n) = scale * u(i,l:n) + + end if + + end if + + x = max ( x, abs ( w(i) ) + abs ( rv1(i) ) ) + + end do +! +! Accumulation of right-hand transformations. +! + if ( matv ) then + + do i = n, 1, -1 + + if ( i /= n ) then + + if ( g /= 0.0D+00 ) then + + v(l:n,i) = ( u(i,l:n) / u(i,l) ) / g + + do j = l, n + + s = dot_product ( u(i,l:n), v(l:n,j) ) + + v(l:n,j) = v(l:n,j) + s * v(l:n,i) + + end do + + end if + + v(i,l:n) = 0.0D+00 + v(l:n,i) = 0.0D+00 + + end if + + v(i,i) = 1.0D+00 + g = rv1(i) + l = i + + end do + + end if +! +! Accumulation of left-hand transformations. +! + if ( matu ) then + + mn = min ( m, n ) + + do i = min ( m, n ), 1, -1 + + l = i + 1 + g = w(i) + + if ( i /= n ) then + u(i,l:n) = 0.0D+00 + end if + + if ( g /= 0.0D+00 ) then + + if ( i /= mn ) then + + do j = l, n + s = dot_product ( u(l:m,i), u(l:m,j) ) + f = ( s / u(i,i) ) / g + u(i:m,j) = u(i:m,j) + f * u(i:m,i) + end do + + end if + + u(i:m,i) = u(i:m,i) / g + + else + + u(i:m,i) = 0.0D+00 + + end if + + u(i,i) = u(i,i) + 1.0D+00 + + end do + + end if +! +! Diagonalization of the bidiagonal form. +! + tst1 = x + + do kk = 1, n + + k1 = n - kk + k = k1 + 1 + its = 0 +! +! Test for splitting. +! +520 continue + + do ll = 1, k + + l1 = k - ll + l = l1 + 1 + tst2 = tst1 + abs ( rv1(l) ) + + if ( tst2 == tst1 ) then + go to 565 + end if + + tst2 = tst1 + abs ( w(l1) ) + + if ( tst2 == tst1 ) then + exit + end if + + end do +! +! Cancellation of rv1(l) if L greater than 1. +! + c = 0.0D+00 + s = 1.0D+00 + + do i = l, k + + f = s * rv1(i) + rv1(i) = c * rv1(i) + tst2 = tst1 + abs ( f ) + + if ( tst2 == tst1 ) then + go to 565 + end if + + g = w(i) + h = pythag ( f, g ) + w(i) = h + c = g / h + s = -f / h + + if ( matu ) then + + do j = 1, m + y = u(j,l1) + z = u(j,i) + u(j,l1) = y * c + z * s + u(j,i) = -y * s + z * c + end do + + end if + + end do +! +! Test for convergence. +! +565 continue + + z = w(k) + + if ( l == k ) then + go to 650 + end if +! +! Shift from bottom 2 by 2 minor. +! + if ( its >= 30 ) then + ierr = k + return + end if + + its = its + 1 + x = w(l) + y = w(k1) + g = rv1(k1) + h = rv1(k) + f = 0.5D+00 * ( ( ( g + z ) / h ) * ( ( g - z ) / y ) + y / h - h / y ) + g = pythag ( f, 1.0D+00 ) + f = x - ( z / x ) * z + ( h / x ) * ( y / ( f + sign ( g, f ) ) - h ) +! +! Next QR transformation. +! + c = 1.0D+00 + s = 1.0D+00 + + do i1 = l, k1 + + i = i1 + 1 + g = rv1(i) + y = w(i) + h = s * g + g = c * g + z = pythag ( f, h ) + rv1(i1) = z + c = f / z + s = h / z + f = x * c + g * s + g = -x * s + g * c + h = y * s + y = y * c + + if ( matv ) then + + do j = 1, n + x = v(j,i1) + z = v(j,i) + v(j,i1) = x * c + z * s + v(j,i) = -x * s + z * c + end do + + end if + + z = pythag ( f, h ) + w(i1) = z +! +! Rotation can be arbitrary if Z is zero. +! + if ( z /= 0.0D+00 ) then + c = f / z + s = h / z + end if + + f = c * g + s * y + x = - s * g + c * y + + if ( matu ) then + + do j = 1, m + y = u(j,i1) + z = u(j,i) + u(j,i1) = y * c + z * s + u(j,i) = - y * s + z * c + end do + + end if + + end do + + rv1(l) = 0.0D+00 + rv1(k) = f + w(k) = x + go to 520 +! +! Convergence. +! +650 continue + + if ( z <= 0.0D+00 ) then + + w(k) = - z + + if ( matv ) then + v(1:n,k) = - v(1:n,k) + end if + + end if + + end do + + return +end + + +function pythag ( a, b ) + +!*****************************************************************************80 +! +!! PYTHAG computes SQRT ( A * A + B * B ) carefully. +! +! Discussion: +! +! The formula +! +! PYTHAG = sqrt ( A * A + B * B ) +! +! is reasonably accurate, but can fail if, for example, A^2 is larger +! than the machine overflow. The formula can lose most of its accuracy +! if the sum of the squares is very large or very small. +! +! Licensing: +! +! This code is distributed under the GNU LGPL license. +! +! Modified: +! +! 18 October 2009 +! +! Author: +! +! Original FORTRAN77 version by Smith, Boyle, Dongarra, Garbow, Ikebe, +! Klema, Moler. +! FORTRAN90 version by John Burkardt. +! +! Reference: +! +! James Wilkinson, Christian Reinsch, +! Handbook for Automatic Computation, +! Volume II, Linear Algebra, Part 2, +! Springer, 1971, +! ISBN: 0387054146, +! LC: QA251.W67. +! +! Brian Smith, James Boyle, Jack Dongarra, Burton Garbow, +! Yasuhiko Ikebe, Virginia Klema, Cleve Moler, +! Matrix Eigensystem Routines, EISPACK Guide, +! Lecture Notes in Computer Science, Volume 6, +! Springer Verlag, 1976, +! ISBN13: 978-3540075462, +! LC: QA193.M37. +! +! Modified: +! +! 04 February 2003 +! +! Parameters: +! +! Input, real ( kind = 8 ) A, B, the two legs of a right triangle. +! +! Output, real ( kind = 8 ) PYTHAG, the length of the hypotenuse. +! + implicit none + + real ( kind = 8 ) a + real ( kind = 8 ) b + real ( kind = 8 ) p + real ( kind = 8 ) pythag + real ( kind = 8 ) r + real ( kind = 8 ) s + real ( kind = 8 ) t + real ( kind = 8 ) u + + p = max ( abs ( a ), abs ( b ) ) + + if ( p /= 0.0D+00 ) then + + r = ( min ( abs ( a ), abs ( b ) ) / p )**2 + + do + + t = 4.0D+00 + r + + if ( t == 4.0D+00 ) then + exit + end if + + s = r / t + u = 1.0D+00 + 2.0D+00 * s + p = u * p + r = ( s / u )**2 * r + + end do + + end if + + pythag = p + + return +end diff --git a/kp_scripts/sentinel/process_parallel.py b/kp_scripts/sentinel/process_parallel.py new file mode 100644 index 0000000..a25eb30 --- /dev/null +++ b/kp_scripts/sentinel/process_parallel.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +# +# +# process_parallel_kp.py -- process multiple scenes in parallel, depending on number of gpus +# +# note: assume that zipfiles exists +# + +import os +import subprocess +import sys + +if len(sys.argv) <2: + print ('Usage: process_parallel_kp.py ') + sys.exit(0) + +# Default Parameters +zipfiles='zipfiles' +ngpus='1' +pol='vv' + +# Get Optional Parameters +if len(sys.argv)>1: + zipfiles = sys.argv[1] + +if len(sys.argv)>2: + ngpus = sys.argv[2] + +if len(sys.argv)>3: + pol = sys.argv[3] + +print (' process_parallel_kp args: ',zipfiles,ngpus) + +# first create multiple lists of zipfiles +command = '$PROC_HOME/util/splitziplist.py '+zipfiles+' '+ngpus # split zips into many lists +print('\n '+command) +ret = os.system(command) + +# how many ziplists do you form? +command = 'ls -1 ziplist* | cat > listofziplists' # split into many lists +print('\n '+command) +ret = os.system(command) + +flist = open('listofziplists','r') +list = flist.readlines() +flist.close() + +##### ---- Process in Parallel ----- ##### +num=0 +unwcommand=[] +prochome = os.getenv('PROC_HOME') +for ziplist in list: + if ziplist.rstrip() != 'ziplist': + if os.path.getsize(ziplist.rstrip()) > 0: + command = prochome+'/sentinel/process_zip_list.py '+ziplist.rstrip()+' '+str(num) + print(' '+command) + command = prochome+'/sentinel/process_zip_list.py' + unwcommand.append(subprocess.Popen([command,ziplist.rstrip(),str(num)])) + num=num+1 + +for i in range(num): + unwcommand[i].wait() + +print('geo files created') + + + + diff --git a/kp_scripts/sentinel/ps_sbas_igrams.py b/kp_scripts/sentinel/ps_sbas_igrams.py new file mode 100644 index 0000000..a696446 --- /dev/null +++ b/kp_scripts/sentinel/ps_sbas_igrams.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 + +# ps_sbas_igrams - create set of sentinel interferogram subsets +# allow looks +# create the intlist +# don't subset if you don't need to + +import sys +import string +import os +import math + +if len(sys.argv) < 6: + print ('Usage: ps_sbas_igrams.py sbas_list dem_rsc_file xstart ystart xsize ysize ') + sys.exit(1) + +sbaslist=sys.argv[1] +demrscfile=sys.argv[2] +xstart=sys.argv[3] +ystart=sys.argv[4] +xsize=sys.argv[5] +ysize=sys.argv[6] +xlooks='1' + +if len(sys.argv) > 7: + xlooks=sys.argv[7] + +ylooks=xlooks +if len(sys.argv) > 8: + ylooks=sys.argv[8] +commonflag='n' + +##### ----- GENERATE A NEW DEM.RSC FILE FOR MULTILOOKED AND/OR EXTRACTED PORTION ----- ##### +if os.path.isfile('dem.rsc') == False: + command = '$PROC_HOME/kp_scripts/util/make_ml_demrsc.py '+demrscfile+' '+xstart+' '+ystart+' '+xsize+' '+ysize+' '+xlooks+' '+ylooks + print('\n '+command) + ret=os.system(command) + +# Get original DEM parameters +with open(demrscfile,'r') as rsc: + words=rsc.readline() + demwidth=words.split()[1] + words=rsc.readline() + demlength=words.split()[1] + +# sbaslist +intlist=sbaslist.replace('sbas','int') +fintlist=open(intlist,'w') + +sbasfiles=[] +fsbas=open(sbaslist,'r') +sbas=fsbas.readlines() +for line in sbas: + words=line.split() + primary=words[0] + secondary=words[1] +# get a short names for primary and secondary files + first=primary.find('20') + primaryname=primary[first:first+8] + first=secondary.find('20') + secondaryname=secondary[first:first+8] + + intfile=primaryname+'_'+secondaryname+'.int' + ampfile=primaryname+'_'+secondaryname+'.amp' + ccfile=primaryname+'_'+secondaryname+'.cc' + + # check to see if the file already exists + fintlist.write(intfile+'\n') + if os.path.isfile(intfile): + continue + + flag=0 + if int(xstart) == 1: + if int(ystart) == 1: + if int(xsize) == int(demwidth): + if int(ysize) == int(demlength): + command='$PROC_HOME/int/crossmul '+primary+' '+secondary+' '+intfile+' '+ampfile+' '+xsize+' '+ysize+' 1.e-6 '+str(xlooks)+' '+str(ylooks) + print (command) + ret=os.system(command) + flag=1 + + if flag == 0: + command='$PROC_HOME/util/subsetmph '+primary+' primarypiece '+demwidth+' '+xstart+' '+ystart+' '+xsize+' '+ysize + print (command) + ret = os.system(command) + command='$PROC_HOME/util/subsetmph '+secondary+' secondarypiece '+demwidth+' '+xstart+' '+ystart+' '+xsize+' '+ysize + print (command) + ret = os.system(command) + command='$PROC_HOME/int/crossmul primarypiece secondarypiece '+intfile+' '+ampfile+' '+xsize+' '+ysize+' 1.e-6 '+str(xlooks)+' '+str(ylooks) + print (command) + ret=os.system(command) + + # correlation file next + ret=os.system('$PROC_HOME/int/makecc '+' '+intfile+' '+ampfile+' '+ccfile+' '+str(int((int(xsize)/int(xlooks))))) + +fintlist.close() + +sys.exit() + diff --git a/kp_scripts/sentinel/sbas_list.py b/kp_scripts/sentinel/sbas_list.py new file mode 100644 index 0000000..daf8801 --- /dev/null +++ b/kp_scripts/sentinel/sbas_list.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 + +# create a list of sbas pairs +# max temporal = 0 generates minimal list of consecutive acquisitions + +import sys +import subprocess +from datetime import datetime +import os + +if len(sys.argv) < 4: + print('Usage: sbas_list_kp.py geolist geopath sbaslist_out max_temporal max_spatial') + sys.exit(1) + +geolist = sys.argv[1] +geopath = sys.argv[2] +sbaslist_out = sys.argv[3] +maxtemporal=float(sys.argv[4]) +maxspatial=float(sys.argv[5]) + +# get a list of the geocoded slc files +with open(geolist,'r') as fgeo: + geos = fgeo.readlines() +for k in range(len(geos)): + geofile = geos[k].split('/')[-1] + geos[k] = geopath+'/'+geofile.strip() + +# sort geofiles by date order +names_times=[] + +for i in range(0,len(geos)): + words=geos[i].split('/')[-1] + wordstring=str(words) + scenedate=words[:8] + + jd=datetime.strptime(str(scenedate), '%Y%m%d').toordinal()+1721424.5 + names_times.append(str(jd)+' '+geos[i]) + +sortedgeos=sorted(names_times) + +# estimate baseline and create a file for the time-baseline plot +ftb=open(sbaslist_out,'w') + +# create lists of dates and filenames, write out geolist +geolist=open(geolist+'_fullpath','w') +jdfile=open('jdlist','w') +jdlist=[] +for i in range(0,len(sortedgeos)): + geos[i]=sortedgeos[i].split()[1] + geolist.write(geos[i]+'\n') + + jdlist.append(float(sortedgeos[i].split()[0])) + jdfile.write(str(jdlist[i])+'\n') + +geolist.close() +jdfile.close() +print ('Julian day range: ',jdlist[0],jdlist[-1]) + +# call the spatial baseline estimator +print ('Estimating baselines...') +if maxtemporal > 0: + + for i in range(0,len(jdlist)): + for j in range(0,i): + # first do temporal filter + baseline2=abs(jdlist[i]-jdlist[j]) + if baseline2 <= maxtemporal: + + # spatial baseline estimator + orbtimingi=geos[i].strip().replace('geo','orbtiming') + orbtimingj=geos[j].strip().replace('geo','orbtiming') + command = '$PROC_HOME/sentinel/geo2rdr/estimatebaseline '+orbtimingi+' '+orbtimingj + proc = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True) + (baseline1, err) = proc.communicate() + #print (command,' baseline1: ',str(baseline1,"UTF-8")) + if abs(float(baseline1)) <= maxspatial: + geostri=geos[i] + geostrj=geos[j] + ftb.write(geostrj+' '+geostri+' '+str(baseline2)+' '+str(baseline1,"UTF-8")) + +else: # zero max temporal means minimal list of pairs + for i in range(1,len(jdlist)): + j=i-1 + orbtimingi=geos[i].strip().replace('geo','orbtiming') + orbtimingj=geos[j].strip().replace('geo','orbtiming') + + baseline2=abs(jdlist[i]-jdlist[j]) + geostri=geos[i] + geostrj=geos[j] + ftb.write(geostrj+' '+geostri+' '+str(baseline2)+' 100000000'+'\n') + +print ('sbas list written') +ftb.close() diff --git a/kp_scripts/sentinel/sentinel_cpu.py b/kp_scripts/sentinel/sentinel_cpu.py new file mode 100644 index 0000000..4cb09c7 --- /dev/null +++ b/kp_scripts/sentinel/sentinel_cpu.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +# +# process stack of sentinel files to coregistered geocoded slcs from L0 data. +# +# Either download the zip files you want to process, then start this script: +# +# sentinel_cpu_kp.py +# +# backproject using cpu, not gpu, integration +# + +import sys +import os +import string +import time +import subprocess +from datetime import datetime + + +if len(sys.argv) < 1: + print ('Usage: sentinel_cpu.py ') + +# Default Parameters +update_flag = 'N' +pol='vv' + +# Get Optional Inputs +if len(sys.argv) > 1: + update_flag = sys.argv[1] + +if len(sys.argv) > 2: + pol=sys.argv[2] + if pol == 'VH': + pol='vh' + +# and start +print ('\n##### ----- Processing stack of sentinel raw data products to coregistered geocoded slcs ----- #####') + +# get the PATH of the script directory +PATH=os.path.dirname(os.path.abspath(sys.argv[0])) + +# get list of L0 products +zipfiles = [] +SAFEnames = [] +num=0 +unzipcommand=[] +for file in os.listdir("."): + if file.endswith(".zip"): + zipfiles.append(file) + unzipcommand.append(subprocess.Popen(['unzip','-u',file])) + SAFEnames.append(file[0:len(file)-4]) + num=num+1 + +# unzip the zip files to safe files if the safefile doesn't already exist +for i in range(num): + if os.path.isfile(SAFEnames[i])==False: + unzipcommand[i].wait() + +# download the precise orbit files for these zips +command = '$PROC_HOME/kp_scripts/sentinel/sentinel_orbitfiles.py '+update_flag +print('\n '+command) +ret=os.system(command) + +# list the precise orbit files +ret=os.system('ls -1 *.EOF | cat > preciseorbitfiles') +with open('preciseorbitfiles','r') as preciseorbitfiles: + preciseorbitlist=preciseorbitfiles.readlines() + +# Load or create list of processed files +if update_flag == 'Y': + processed = [] + fproc = open('processed','w') +else: + if os.path.isfile('processed')==False: + fproc = open('processed','w') + processed = [] + else: + with open('processed','r') as fproc: + processed = fproc.readlines() + fproc = open('processed','a') + +# loop over directories and process each with sentinel_back.py +# sentinel_back needs zipfile and precise orbit +print('\n##### ----- Processing Sentinel Scene ----- #####') +for zipfile in zipfiles: + print(zipfile) + # which precise orbit file for this scene? + if (zipfile+'\n' in processed) and (update_flag == 'N'): + print('\n skipping '+zipfile.strip()) + continue + print ('\n zipfile: ',zipfile) + # H or V pol + char1=zipfile.find('SSV_') + if char1 < 0: + char1=zipfile.find('SDV_') + pol='vh' + print (' This is a dual pol acquisition '+pol) + else: + pol='vv' + print (' This is a single pol acquisition '+pol) + if char1 < 0: + char1=zipfile.find('SSH_') + if char1 < 0: + char1=zipfile.find('SDH_') + pol='vh' + print (' This is a dual pol acquisition '+pol) + else: + pol='hh' + print (' This is a single pol acquisition '+pol) + char2=zipfile[char1:].find('T') + scenedate=zipfile[char1+4:char1+char2] + + doy=datetime.strptime(scenedate, '%Y%m%d').timetuple().tm_yday + year=scenedate[0:4] # day of year and year for scene + print (' doy ',doy,' ',year) + if doy > 1: + orbitfilestartdate = datetime.strptime(year+' '+str(doy-1),'%Y %j').strftime('%Y%m%d') + else: + lastdoy=datetime.strptime(str(int(year)-1)+'1231', '%Y%m%d').timetuple().tm_yday + orbitfilestartdate = datetime.strptime(str(int(year)-1)+' '+str(lastdoy),'%Y %j').strftime('%Y%m%d') + command='grep '+orbitfilestartdate+' preciseorbitfiles' + proc = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True) + (orbitfilename, err) = proc.communicate() + orbitfilename=str(orbitfilename,'UTF-8').rstrip() + SAFEname=zipfile[0:len(zipfile)-4] + + +# and process the scene if you have the precise orbit and it hasn't already been processed + if len(orbitfilename) < 1: + print (' Skipping: Missing precise orbit file for ',zipfile) + else: + command='$PROC_HOME/kp_scripts/sentinel/sentinel_scene_cpu.py '+SAFEname+' '+orbitfilename+' ../params '+pol + print(' '+command) + ret = os.system(command) + fproc.write(zipfile+'\n') +fproc.close() + +print (' Loop over scenes complete.') + +# clean up a bit +command = 'rm *positionburst*.out' +ret=os.system(command) + + diff --git a/kp_scripts/sentinel/sentinel_gpu.py b/kp_scripts/sentinel/sentinel_gpu.py new file mode 100644 index 0000000..569962c --- /dev/null +++ b/kp_scripts/sentinel/sentinel_gpu.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 +# +# process stack of sentinel files to coregistered geocoded slcs from L0 data +# +# First, either download the zip files you want to process, then start this script: +# +# sentinel_gpu_cloud.py +# +# backproject using gpu integration +# +# If not logged in you willbe prompted for username/password in vertex/Earthdata system +# + +import sys +import os +import string +import time +import subprocess +from datetime import datetime + +if len(sys.argv) < 1: + print ('Usage: sentinel_gpu.py ') + +# Default Parameters +update_flag = 'N' +pol='vv' + +# Get optional inputs +if len(sys.argv) > 1: + update_flag = sys.argv[1] + +if len(sys.argv) > 2: + pol=sys.argv[2] + if pol == 'VH': + pol='vh' + +# and start +print ('\n##### ----- Processing stack of sentinel raw data products to coregistered geocoded slcs ----- #####') + +# get the PATH of the script directory +PATH=os.path.dirname(os.path.abspath(sys.argv[0])) + +# Load or create list of processed files +if update_flag == 'Y': + processed = [] + fproc = open('processed','w') +else: + if os.path.isfile('processed'): + with open('processed','r') as fproc: + processed = fproc.readlines() + fproc = open('processed','a') + else: + processed = [] + fproc = open('processed','w') + +# get list of L0 products that need to be processed (or updated, if update_flag = 'Y') +fziplist=open('zipfiles','w') +zipfiles = [] +SAFEnames = [] +num=0 +unzipcommand=[] +for file in os.listdir("."): + if file.endswith(".zip"): + zipfile = file.strip() + if (zipfile+'\n' in processed) and (update_flag == 'N'): + print('\n skipping '+zipfile.strip()) + continue + print('\n '+zipfile) + zipfiles.append(file) + unzipcommand.append(subprocess.Popen(['unzip', '-u',file])) + SAFEnames.append(file[0:len(file)-4]) + fziplist.write(file+'\n') + num=num+1 + +# unzip the zip files to safe files is the safefile doesn't already exist +for i in range(num): + if os.path.isfile(SAFEnames[i])==False: + unzipcommand[i].wait() + +fziplist.close() + +if len(zipfiles)>0: + # download the precise orbit files for these zips + command = PATH+'/sentinel_orbitfiles.py '+update_flag + print('\n '+command) + ret=os.system(command) + + # list the precise orbit files + ret=os.system('ls -1 *.EOF | cat > preciseorbitfiles') + with open('preciseorbitfiles','r') as preciseorbitfiles: + preciseorbitlist=preciseorbitfiles.readlines() + + + ##### ----- Process in Parallel ----- ##### + # how many gpus do we have? + proc = subprocess.Popen("$PROC_HOME/sentinel/howmanygpus",stdout=subprocess.PIPE, shell=True) + (param,err)=proc.communicate() + ngpus=str(param,'UTF-8').split()[0] + print ('gpus available: ',ngpus) + + command='$PROC_HOME/kp_scripts/sentinel/process_parallel.py zipfiles '+str(ngpus) + print (command) + ret=os.system(command) + + print ('\n Loop over scenes complete.') + + # Write list of zipfiles to 'processed' + for filename in zipfiles: + if os.path.isfile(filename.replace('.zip','.geo')): + fproc.write(filename+'\n') + fproc.close() + + # clean up a bit + command = 'find . -name \*positionburst\* -delete' + ret=os.system(command) + + + + + + + diff --git a/kp_scripts/sentinel/sentinel_orbitfiles.py b/kp_scripts/sentinel/sentinel_orbitfiles.py new file mode 100644 index 0000000..f14430f --- /dev/null +++ b/kp_scripts/sentinel/sentinel_orbitfiles.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +# +# download precise orbit files for zipfiles in directory +# + +import sys +import os +import string +import time +import subprocess +from datetime import datetime + +if len(sys.argv) < 1: + print ('Usage: sentinel_orbitfiles_kp.py ') + +update_flag = 'N' +if len(sys.argv) > 1: + update_flag = sys.argv[1] + +# get list of zip files +filename = 'processed' # A file that tracks whether the corresponsing SLC has been generated. If so, it's EOF file has already been downloaded +eofs=[] +if os.path.isfile(filename): + with open(filename,'r') as feof: + eofs=feof.readlines() + +print(' Downloading precise orbit files from ASF, if needed') +zipfiles = [] +for file in os.listdir("."): + if file.endswith(".zip"): + SAFEname = file.replace('.zip','\n') + if (SAFEname in eofs) and (update_flag=='N'): + continue + + command = 'wget --content-disposition https://s1-orbits.asf.alaska.edu/scene/'+SAFEname.strip() + print(' '+command) + ret=os.system(command) + +# Clean duplicates +command = 'rm *.EOF.*' +ret=os.system(command) + +# Remove non-precise orbit files, because we don't want these to be processed until the best file is available +command = 'rm *RESORB*' +ret=os.system(command) + +print (' EOF (precise orbits) files downloaded!') diff --git a/kp_scripts/sentinel/sentinel_scene_cpu.py b/kp_scripts/sentinel/sentinel_scene_cpu.py new file mode 100644 index 0000000..93317bb --- /dev/null +++ b/kp_scripts/sentinel/sentinel_scene_cpu.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +# +# do the decoding and sorting of raw data for sentinel backprojection processing + +import sys +import os +import subprocess + +if len(sys.argv) < 4: + print ('Usage: sentinel_scene_cpu.py SAFEname orbitfile(*EOF) params_file ') + +print ('Processing stack of sentinel raw data products to coregistered geocoded slcs') + +SAFEname=sys.argv[1] +orbitfile=sys.argv[2] +params=sys.argv[3] +pol='vv' +if len(sys.argv) > 4: + pol=sys.argv[4] + if pol == 'VH': + pol='vh' + + print ('Processing ',pol,' polarization') + + +# find the basename +command = 'ls '+SAFEname+'.SAFE/*'+pol+'*dat | grep -v annot | grep -v index' +print (command) +proc = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True) +(datfile, err) = proc.communicate() +basename=str(datfile[0:len(datfile)-5],'UTF-8') + +# read the orbitfile statevectors, store in orbtiming.full +command = '$PROC_HOME/sentinel/orbitstatevectors.py '+orbitfile+' '+SAFEname +print (command) +ret=os.system(command) + +# extract path to DEM and resource file +fparam = open(params,'r') +demfile = fparam.readline().strip() +rscfile = fparam.readline().strip() +fparam.close() + +# initialize the slc file +fe=open(rscfile,'r') +words=fe.readline() +demwidth=words.split()[1] +words=fe.readline() +demlength=words.split()[1] +fe.close() + +command = '$PROC_HOME/sentinel/createslc '+demwidth+' '+demlength+' '+SAFEname+'.geo' +print (command) +ret=os.system(command) + +# process the scene, three swaths +command = '$PROC_HOME/sentinel//sentinel_raw_process_cpu '+basename +print (command) +ret=os.system(command) + +sys.exit(0) + + diff --git a/kp_scripts/util/coverage_mask b/kp_scripts/util/coverage_mask new file mode 100644 index 0000000..2b499aa Binary files /dev/null and b/kp_scripts/util/coverage_mask differ diff --git a/kp_scripts/util/coverage_mask.f90 b/kp_scripts/util/coverage_mask.f90 new file mode 100644 index 0000000..7e57f62 --- /dev/null +++ b/kp_scripts/util/coverage_mask.f90 @@ -0,0 +1,70 @@ +! Determine where the satellite acquired valid data (orbital coverage) +! Read in an SLC (or multilooked SLC) and find if data was acquired pixel by +! pixel (in parallel) + +! Compile as: gfortran -o coverage_mask coverage_mask.f90 + +PROGRAM coverage_mask + + IMPLICIT none + + !specifications + INTEGER::i,j + INTEGER*8 ::nr, naz + CHARACTER(200)::str + CHARACTER(200)::filename,outfile + REAL,DIMENSION(:,:),ALLOCATABLE::amp + complex*8,dimension(:,:),allocatable :: slc + integer*1,dimension(:,:),allocatable :: bytemask + + !executions + + !slcfile: name of SLC file to load + !nr: number of range bins + !naz: number of azimuth bins + !outfile: name of output + IF(iargc().lt.3)THEN + WRITE(*,*)'usage: coverage_mask slcfile len wid outfile' + STOP + END IF + + CALL getarg(1,filename) + CALL getarg(2,str) + READ(str,*)nr !width of file + CALL getarg(3,str) + READ(str,*)naz !length of file + CALL getarg(4,outfile) + + + ! Allocate arrays + ALLOCATE(amp(nr,naz),bytemask(nr,naz)) + ALLOCATE(slc(nr,naz)) + + ! Load in the SLC file (filename) + print *,'Reading ',filename + OPEN(UNIT=10,FILE=filename,FORM='unformatted',ACCESS='direct',RECL=2*nr*naz*4,CONVERT='LITTLE_ENDIAN') + READ(10,rec=1)slc ! read in complex SLC + CLOSE(10) + amp=cabs(slc)**2 ! calculate the amplitude (power) from SLC + + ! Create the mask given the SLC amplitude + print *,'Calculating mask' + bytemask=0 + do j=1,naz + do i=1,nr + if(amp(i,j).ne.0)then + bytemask(i,j)=-1 + end if + end do + end do + + ! Write out the make to outfile + open(unit=20,file=outfile,form='unformatted',status='replace',access='direct',recl=nr*naz) + write(20,rec=1)bytemask + close(20) + print *,'Mask saved to ',outfile + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +END PROGRAM coverage_mask + diff --git a/kp_scripts/util/cull_geolist_fromsim b/kp_scripts/util/cull_geolist_fromsim new file mode 100644 index 0000000..476000f Binary files /dev/null and b/kp_scripts/util/cull_geolist_fromsim differ diff --git a/kp_scripts/util/cull_geolist_fromsim.f90 b/kp_scripts/util/cull_geolist_fromsim.f90 new file mode 100644 index 0000000..47e1ea2 --- /dev/null +++ b/kp_scripts/util/cull_geolist_fromsim.f90 @@ -0,0 +1,132 @@ +! run ps detection using cosine similarlity on a set of wrapped files +! Read also correlation and amp files +! Read also mask files (if designated), in order to account for inconsistent +! coverage +! this version allows for averaging multiple reference points if file is supplied +! modified to use fftw3 28jan23 + +! Compile as: gfortran -o cull_geolist_fromsim cull_geolist_fromsim.f90 -fopenmp + +PROGRAM cull_geolist_fromsim + use omp_lib + + IMPLICIT none + + !!!!! ----- Specifications: Declare your variables ----- !!!!! + INTEGER::stat + INTEGER*8::nr,naz,nslc,i + CHARACTER(200),DIMENSION(:),ALLOCATABLE::geonames + CHARACTER(200)::geolist,geolist_out,str,scenemaskfile,strgeo,strmask + REAL*8,DIMENSION(:),ALLOCATABLE::prct + REAL*8::thresh,nvalid,nvalid_temp + INTEGER*1,DIMENSION(:,:,:),ALLOCATABLE::masks + INTEGER*1,DIMENSION(:,:),ALLOCATABLE::tempmask,scenemask + + + !!!!! ----- Executions ----- !!!!! + + ! geolist: input geolist to check its all_similarity_masks + ! nr: width of the files + ! naz: number of lines in the files + ! nslc: number of slcs in geolist + ! geolist_out: name of output geolist + ! scenemaskfile: name of input scenemask + ! thresh (optional): threshold percent to be included in final geolist (default = 0.1) + + IF(iargc().lt.6)THEN + WRITE(*,*)'usage: cull_geolist_fromsim geolist nr naz nslc geolist_out scenemaskfile ' + STOP + END IF + + CALL getarg(1,geolist) + CALL getarg(2,str) + READ(str,*)nr !width of files + CALL getarg(3,str) + READ(str,*)naz + CALL getarg(4,str) + READ(str,*)nslc + call getarg(5,geolist_out) + call getarg(6,scenemaskfile) + + thresh = 0.1 + if(iargc().ge.7)then + call getarg(7,str) + read(str,*)thresh + end if + + !!!!! ----- ALLOCATE THE VARIABLES ----- !!!!! + ALLOCATE(masks(nr,naz,nslc)) + ALLOCATE(geonames(nslc)) + ALLOCATE(scenemask(nr,naz)) + + !!!!! ----- READ IN THE LIST OF .GEO FILES ----- !!!!! + OPEN(UNIT=11,FILE=geolist,STATUS='old') + DO i=1,nslc + READ(11,'(A)',end=111,IOSTAT=stat)geonames(i) + END DO + 111 continue + CLOSE(11) + + !!!!! ----- READ IN THE REFERENCE SCENEMASK ----- !!!!! + OPEN(UNIT=11,file=scenemaskfile,access='stream') + READ(11)scenemask + CLOSE(11) + + !!!!! ----- Determine the total number of potentially-valid pixels ----- ##### + nvalid = sum(abs(real(scenemask))) + + !!!!! ----- READ IN THE SLC MASKS IN PARALLEL ----- !!!!! + ALLOCATE(prct(nslc)) + + !$OMP parallel do private(strgeo,strmask,tempmask,nvalid_temp) & + !$OMP shared(geonames,masks,prct,nvalid) + DO i=1,nslc + ALLOCATE(tempmask(nr,naz)) + + strgeo=trim(adjustl(geonames(i))) + strmask='all_similarity_mask_' // strgeo(:8) + + OPEN(UNIT=i+10,FILE=strmask,access='stream') + READ(i+10)tempmask + CLOSE(i+10) + masks(:,:,i) = tempmask + + nvalid_temp = sum(abs(real(masks(:,:,i)))) + prct(i) = nvalid_temp/nvalid + + DEALLOCATE(tempmask) + END DO + !$OMP end parallel do + + + !!!!! ----- WRITE OUT THE OUTPUT GEOLIST ----- !!!!! + OPEN(UNIT=21,FILE=geolist_out,status='unknown',ACTION='write') + DO i=1,nslc + if (prct(i).ge.thresh)then + WRITE(21,'(A)') trim(geonames(i)) + end if + END DO + CLOSE(21) + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +!FUNCTIONS + +CONTAINS + +! ------------------ +FUNCTION Replace_Text (s,text,rep) RESULT(outs) +CHARACTER(*) :: s,text,rep +CHARACTER(LEN(s)+100) :: outs ! provide outs with extra 100 char len +INTEGER :: i, nt, nr + +outs = s ; nt = LEN_TRIM(text) ; nr = LEN_TRIM(rep) +DO + i = INDEX(outs,text(:nt)) ; IF (i == 0) EXIT + outs = outs(:i-1) // rep(:nr) // outs(i+nt:) +END DO +END FUNCTION Replace_Text + + +END PROGRAM cull_geolist_fromsim + diff --git a/kp_scripts/util/determine_valid b/kp_scripts/util/determine_valid new file mode 100644 index 0000000..a80de3e Binary files /dev/null and b/kp_scripts/util/determine_valid differ diff --git a/kp_scripts/util/determine_valid.f90 b/kp_scripts/util/determine_valid.f90 new file mode 100644 index 0000000..dec1481 --- /dev/null +++ b/kp_scripts/util/determine_valid.f90 @@ -0,0 +1,425 @@ +! Determine valid geocoded slcs and pixels within the scene. +! This will result in two new geolists: +! geolist_consistent = list of SLCs that share consistent coverage. Script +! maximizes number of SLCs with the same coverage (could be improved to +! optimize temporal & spatial coverage) +! +! geolist_culled = list of SLCs that cover at least thresh% of the valid +! study area. The valid area is the union of all scenes' coverage (determined +! from the input masks). The default threshold is 30% of valid. +! +! The script also outputs summation maps (The number of valid acquisitions at +! each pixel): +! sum_coverage_geolist_ml (based on all geocoded slcs that have been +! processed) +! +! sum_coverage_geolist_consistent (based on geolist_consistent) +! +! sum_coverage_geolist_culled (based on geolist_culled) +! +! The script will also output the corresponding binary masks: +! scene_mask_ml = indicates where sum_coverage_geolist_ml > 1 +! +! scene_mask_consistent = indicates where sum_coverage_geolist_consistent +! equals the number of consistent SLCs +! +! scene_mask_culled = indicates where sum_coverage_geolist_culled > 1 + +! NOTE: the script could be improved by excluding valid scene edges (e.g. far +! range or close range), since these are wildly inconsistent and we don't expect +! the accuracy at these locations to be very high. + + +! Compile with: +! gfortran -o determine_valid determine_valid.f90 -fopenmp -lrt -lpthread + +PROGRAM determine_valid + use omp_lib + + IMPLICIT none + + !!!!! SPECIFICATIONS !!!!! + + ! INPUTS + INTEGER:: nslc + INTEGER*8::nr,naz + CHARACTER(100)::geolist + REAL::thresh + + ! ERROR/STATUS CHECKS/COUNTING VARIABLES + INTEGER::k,j,stat,ierr,stat2,ios + INTEGER,DIMENSION(13)::statb + CHARACTER(100)::str + + ! READING IN INPUTS LISTS AND DEFINING OTHER VARIABLES + CHARACTER(100),DIMENSION(:),ALLOCATABLE::geonames + + ! DATA ARRAYS FOR INPUT MASKS + INTEGER,DIMENSION(:,:,:),ALLOCATABLE::scenemasks + INTEGER*1,DIMENSION(:,:),ALLOCATABLE::mask + INTEGER,DIMENSION(:,:),ALLOCATABLE::mask_count + + ! LOADING IN THE MASKS + CHARACTER(100)::strgeo,strmask + + ! ARRAYS FOR SUMMING VALID ACQUISITIONS + INTEGER,DIMENSION(:,:),ALLOCATABLE::sum_full,sum_consistent,sum_culled + REAL,DIMENSION(:,:),ALLOCATABLE::temp_array + + ! ARRAYS FOR IDENTIFYING POTENTIALLY VALID PIXELS (MASKS) + INTEGER,DIMENSION(:,:),ALLOCATABLE::valid_full,valid_consistent,valid_culled + INTEGER*8::nvalid_full,nvalid_consistent,nvalid_culled + + ! FOR DETERMINING COVERAGE GROUPS + INTEGER,DIMENSION(:),ALLOCATABLE::pixel_counts,counts,counts_groups + REAL,DIMENSION(:),ALLOCATABLE::prct_of_valid,bins,bins_groups + REAL::dp + INTEGER::nslc_consistent,nslc_culled + INTEGER, DIMENSION(1) :: max_idx + INTEGER::n,idx_consistent,counter,n_groups + INTEGER,DIMENSION(:),ALLOCATABLE::index_consistent,index_culled + + ! NAMES FOR OUTPUT FILES + CHARACTER(100)::geolist_consistent,geolist_culled + CHARACTER(100)::sum_full_out,sum_consistent_out,sum_culled_out + CHARACTER(100)::scenemask_full_out,scenemask_consistent_out,scenemask_culled_out + + ! ARRAYS FOR OUTPUT GEOLISTS + CHARACTER(100),DIMENSION(:),ALLOCATABLE::geonames_consistent,geonames_culled + + ! MASK OUTPUT ARRAYS + INTEGER*1,DIMENSION(:,:),ALLOCATABLE::mask_full,mask_consistent,mask_culled + + + !!!!! EXECUTIONS !!!!! + ! check to see if proper number of arguments were provided + IF(iargc().lt.4)THEN + WRITE(*,*)'usage: determine_valid geolist nslc nr naz ' + STOP + END IF + + ! read in arguments and convert to appropriate type (where necessary) + CALL getarg(1,geolist) + CALL getarg(2,str) + READ(str,*)nslc + CALL getarg(3,str) + READ(str,*)nr + CALL getarg(4,str) + READ(str,*)naz + + thresh=30 + IF(iargc().gt.4)THEN + CALL getarg(5,str) + READ(str,*)thresh + END IF + + ! Allocate the list of .geo files and the scenemasks array + ALLOCATE(geonames(nslc)) + ALLOCATE(scenemasks(nr,naz,nslc)) + + ! Read in the geolist + OPEN(unit=11,FILE=geolist,STATUS='old') + READ(11,'(A)',end=110,IOSTAT=stat2)geonames + 110 continue + CLOSE(11) + + !!!!! ----- READ IN ALL MASKS AND SUMMARIZE ----- !!!!! + + scenemasks=0 + ! Read in the scene masks + !$OMP parallel do private(strgeo,strmask,mask) & + !$OMP shared(geonames,scenemasks) + DO k=1,nslc + ALLOCATE(mask(nr,naz)) + strgeo = trim(adjustl(geonames(k))) + strmask = Replace_Text(strgeo,'.geo','.mask') + + ! Read the corresponding bytemask file + open(unit=k+10,file=strmask,access='stream') + read(k+10)mask + CLOSE(k+10) + scenemasks(:,:,k)=mask + + DEALLOCATE(mask) + END DO + !$OMP end parallel do + + ! Get the sum of acquisitions at each pixel + ALLOCATE(sum_full(nr,naz)) + sum_full=0 + sum_full=sum(abs(scenemasks),dim=3) + + ! Summarize Potentially-Valid Pixels + ALLOCATE(valid_full(nr,naz)) + valid_full = 0 + !$OMP parallel do private(j) shared(valid_full,sum_full) + DO k=1,nr + DO j=1,naz + if (sum_full(k,j) >= 1) then + valid_full(k,j) = 1 + end if + END DO + END DO + !$OMP end parallel do + + nvalid_full = sum(valid_full(:,:)) + print *,' Number of Potentially Valid Pixels Across All Scenes: ',nvalid_full + + + !!!!! ----- FIND COVERAGE GROUPS ----- !!!!! + + ! Determine the number of valid pixels in each scene + ALLOCATE(pixel_counts(nslc)) + !$OMP parallel do private(mask_count) shared(pixel_counts,scenemasks) + DO k=1,nslc + ALLOCATE(mask_count(nr,naz)) + mask_count = scenemasks(:,:,k) + pixel_counts(k) = sum(abs(mask_count)) + DEALLOCATE(mask_count) + END DO + !$OMP end parallel do + + ! Determine the percentage of valid pixels, relative to the number of pixels + ! covered by at least one scene + ALLOCATE(prct_of_valid(nslc)) + prct_of_valid = real(pixel_counts)/real(nvalid_full)*100.0 + + ! Bin the coverage by 0.5% windows + dp = 0.5 !step size for percentage bins + n = int((100.0 - dp) / dp) + 1 ! number of bins + + ALLOCATE(bins(n)) + ! Fill the bins with percentage + DO k=1,n + bins(k) = (k-1) * dp + END DO + + ! Find the number of scenes that correspond to each percentage bin + ALLOCATE(counts(n)) + counts=0 + DO k=1,nslc + DO j=1,n + IF (j/=n) THEN + IF (prct_of_valid(k).ge.bins(j).and.prct_of_valid(k).lt.bins(j)+dp)THEN + counts(j)=counts(j)+1 + exit + END IF + ELSE + IF(prct_of_valid(k).ge.bins(j).and.prct_of_valid(k).le.bins(j)+dp)THEN + counts(j)=counts(j)+1 + END IF + END IF + END DO + END DO + + ! List Coverage Groups + print*,'' + print*,' VALID COVERAGE GROUPS' + n_groups = 0 + DO k=1,n + if (counts(k)>0.and.bins(k).ge.thresh) then + print*,'' + print*,' Percentage Bin: ',bins(k),' - ',bins(k)+dp + print*,' Number of SLCs: ',counts(k) + n_groups = n_groups+1 + end if + END DO + print*,'' + print*,' Number of groups: ',n_groups + + ! Capture Coverage Groups with valid coverage greater than input threshold + ALLOCATE(bins_groups(n_groups)) + ALLOCATE(counts_groups(n_groups)) + counter=1 + DO k=1,n + if (counts(k)>0.and.bins(k).ge.thresh) then + bins_groups(counter) = bins(k) + counts_groups(counter) = counts(k) + counter=counter+1 + end if + END DO + + print*,bins_groups + + !!!!! ----- CREATE ARRAYS AND GEOLIST FOR CONSISTENT COVERAGE ----- !!!!! + + ! Find the percentage bin corresponding to the maximum number of SLCs + nslc_consistent = maxval(counts) + max_idx = maxloc(counts) + idx_consistent = max_idx(1) + + print*,'' + print*,' Consistent Coverage Group: ',bins(idx_consistent),' - ',bins(idx_consistent)+dp + print*,' Number of SLCS: ',nslc_consistent + + ! Consistent Geolist + geolist_consistent = 'geolist_consistent' + ALLOCATE(geonames_consistent(nslc_consistent)) + ALLOCATE(index_consistent(nslc_consistent)) + counter=1 + DO k=1,nslc + if (idx_consistent/=n) THEN + if (prct_of_valid(k).ge.bins(idx_consistent).and.prct_of_valid(k).lt.bins(idx_consistent)+dp) then + geonames_consistent(counter) = geonames(k) + index_consistent(counter) = k + counter=counter+1 + end if + else + if (prct_of_valid(k).ge.bins(n).and.prct_of_valid(k).le.bins(n)+dp) then + geonames_consistent(counter) = geonames(k) + index_consistent(counter) = k + counter=counter+1 + end if + end if + END DO + + + ! Consistent Summation and Mask + ALLOCATE(sum_consistent(nr,naz)) + ALLOCATE(valid_consistent(nr,naz)) + + sum_consistent = sum(abs(scenemasks(:,:,index_consistent)),dim=3) + valid_consistent = 0 + !$OMP parallel do private(j) shared(sum_consistent,valid_consistent) + DO k=1,nr + DO j=1,naz + if (sum_consistent(k,j) == nslc_consistent) then + valid_consistent(k,j) = 1 + end if + END DO + END DO + !$OMP end parallel do + + !!!!! ----- CREATE ARRAYS AND GEOLIST FOR CULLED COVERAGE ----- !!!!! + + ! Find the percentage bin corresponding to the first group with coverage > + ! input threshold + nslc_culled = sum(counts_groups) + + print*,'' + print*,' Culled Coverage Minimum Valid: ',bins_groups(1) + print*,' Number of SLCS: ',nslc_culled + + + ! Culled Geolist + geolist_culled = 'geolist_culled' + ALLOCATE(geonames_culled(nslc_culled)) + ALLOCATE(index_culled(nslc_culled)) + counter=1 + DO k=1,nslc + if (prct_of_valid(k).ge.bins_groups(1)) then + geonames_culled(counter) = geonames(k) + index_culled(counter) = k + counter=counter+1 + end if + END DO + + ! Culled Summation and Mask + ALLOCATE(sum_culled(nr,naz)) + ALLOCATE(valid_culled(nr,naz)) + + sum_culled = sum(abs(scenemasks(:,:,index_culled)),dim=3) + valid_culled = 0 + !$OMP parallel do private(j) shared(sum_culled,valid_culled) + DO k=1,nr + DO j=1,naz + if (sum_culled(k,j)>=2) then + valid_culled(k,j) = 1 + end if + END DO + END DO + !$OMP end parallel do + + !!!!! ----- MAKE THE OUTPUT MASKS ----- !!!!! + ALLOCATE(mask_full(nr,naz),mask_consistent(nr,naz),mask_culled(nr,naz)) + !$OMP parallel do private(j) & + !$OMP shared(valid_full,valid_consistent,valid_culled,mask_full,mask_consistent,mask_culled) + DO k=1,nr + DO j=1,naz + if (valid_full(k,j)==1) then + mask_full(k,j)=-1 + end if + if (valid_consistent(k,j)==1) then + mask_consistent(k,j)=-1 + end if + if (valid_culled(k,j)==1) then + mask_culled(k,j) = -1 + end if + END DO + END DO + !$OMP end parallel do + + + !!!!! ----- WRITE OUT GEOLISTS, SUMS, AND MASKS ----- !!!!! + print *,'' + print *,' Writing out files...' + + ! Geolists + OPEN(28,FILE=geolist_culled,status='unknown',ACTION='write') + DO k=1,nslc_culled + WRITE(28,'(A)') trim(geonames_culled(k)) + END DO + CLOSE(28) + + OPEN(28,FILE=geolist_consistent,status='unknown',ACTION='write') + DO k=1,nslc_consistent + WRITE(28,'(A)') trim(geonames_consistent(k)) + END DO + CLOSE(28) + + ! Summation Arrays + sum_full_out = 'num_scenes_full.out' + sum_consistent_out = 'num_scenes_consistent.out' + sum_culled_out = 'num_scenes_culled.out' + + OPEN(28,FILE=sum_full_out,FORM='unformatted',ACCESS='stream') + WRITE(28)sum_full + CLOSE(28) + + OPEN(28,FILE=sum_consistent_out,FORM='unformatted',ACCESS='stream') + WRITE(28)sum_consistent + CLOSE(28) + + OPEN(28,FILE=sum_culled_out,FORM='unformatted',ACCESS='stream') + WRITE(28)sum_culled + CLOSE(28) + + ! Masks + scenemask_full_out = 'mask_full' + scenemask_consistent_out = 'mask_consistent' + scenemask_culled_out = 'mask_culled' + + OPEN(28,FILE=scenemask_full_out,FORM='unformatted',ACCESS='stream') + WRITE(28)mask_full + CLOSE(28) + + OPEN(28,FILE=scenemask_consistent_out,FORM='unformatted',ACCESS='stream') + WRITE(28)mask_consistent + CLOSE(28) + + OPEN(28,FILE=scenemask_culled_out,FORM='unformatted',ACCESS='stream') + WRITE(28)mask_culled + CLOSE(28) + + PRINT*,'Data written' + + + !!!!! FUNCTIONS !!!!! + + CONTAINS + + ! ------------------ Replace_Text ----------------------- ! + FUNCTION Replace_Text (s,text,rep) RESULT(outs) + CHARACTER(*) :: s,text,rep + CHARACTER(LEN(s)+100) :: outs ! provide outs with extra 100 char len + INTEGER :: i, nt, nr + + outs = s ; nt = LEN_TRIM(text) ; nr = LEN_TRIM(rep) + DO + i = INDEX(outs,text(:nt)) ; IF (i == 0) EXIT + outs = outs(:i-1) // rep(:nr) // outs(i+nt:) + END DO + END FUNCTION Replace_Text + + +END PROGRAM determine_valid diff --git a/kp_scripts/util/lsq.f90 b/kp_scripts/util/lsq.f90 new file mode 100644 index 0000000..9e107b5 --- /dev/null +++ b/kp_scripts/util/lsq.f90 @@ -0,0 +1,992 @@ +MODULE lsq + +! Module for unconstrained linear least-squares calculations. +! The algorithm is suitable for updating LS calculations as more +! data are added. This is sometimes called recursive estimation. +! Only one dependent variable is allowed. +! Based upon Applied Statistics algorithm AS 274. +! Translation from Fortran 77 to Fortran 90 by Alan Miller. +! A function, VARPRD, has been added for calculating the variances +! of predicted values, and this uses a subroutine BKSUB2. + +! Version 1.14, 19 August 2002 - ELF90 compatible version +! Author: Alan Miller +! e-mail : amiller @ bigpond.net.au +! WWW-pages: http://www.ozemail.com.au/~milleraj +! http://users.bigpond.net.au/amiller/ + +! Bug fixes: +! 1. In REGCF a call to TOLSET has been added in case the user had +! not set tolerances. +! 2. In SING, each time a singularity is detected, unless it is in the +! variables in the last position, INCLUD is called. INCLUD assumes +! that a new observation is being added and increments the number of +! cases, NOBS. The line: nobs = nobs - 1 has been added. +! 3. row_ptr was left out of the DEALLOCATE statement in routine startup +! in version 1.07. +! 4. In COV, now calls SS if rss_set = .FALSE. 29 August 1997 +! 5. In TOLSET, correction to accomodate negative values of D. 19 August 2002 + +! Other changes: +! 1. Array row_ptr added 18 July 1997. This points to the first element +! stored in each row thus saving a small amount of time needed to +! calculate its position. +! 2. Optional parameter, EPS, added to routine TOLSET, so that the user +! can specify the accuracy of the input data. +! 3. Cosmetic change of lsq_kind to dp (`Double precision') +! 4. Change to routine SING to use row_ptr rather than calculate the position +! of first elements in each row. + +! The PUBLIC variables are: +! dp = a KIND parameter for the floating-point quantities calculated +! in this module. See the more detailed explanation below. +! This KIND parameter should be used for all floating-point +! arguments passed to routines in this module. + +! nobs = the number of observations processed to date. +! ncol = the total number of variables, including one for the constant, +! if a constant is being fitted. +! r_dim = the dimension of array r = ncol*(ncol-1)/2 +! vorder = an integer vector storing the current order of the variables +! in the QR-factorization. The initial order is 0, 1, 2, ... +! if a constant is being fitted, or 1, 2, ... otherwise. +! initialized = a logical variable which indicates whether space has +! been allocated for various arrays. +! tol_set = a logical variable which is set when subroutine TOLSET has +! been called to calculate tolerances for use in testing for +! singularities. +! rss_set = a logical variable indicating whether residual sums of squares +! are available and usable. +! d() = array of row multipliers for the Cholesky factorization. +! The factorization is X = Q.sqrt(D).R where Q is an ortho- +! normal matrix which is NOT stored, D is a diagonal matrix +! whose diagonal elements are stored in array d, and R is an +! upper-triangular matrix with 1's as its diagonal elements. +! rhs() = vector of RHS projections (after scaling by sqrt(D)). +! Thus Q'y = sqrt(D).rhs +! r() = the upper-triangular matrix R. The upper triangle only, +! excluding the implicit 1's on the diagonal, are stored by +! rows. +! tol() = array of tolerances used in testing for singularities. +! rss() = array of residual sums of squares. rss(i) is the residual +! sum of squares with the first i variables in the model. +! By changing the order of variables, the residual sums of +! squares can be found for all possible subsets of the variables. +! The residual sum of squares with NO variables in the model, +! that is the total sum of squares of the y-values, can be +! calculated as rss(1) + d(1)*rhs(1)^2. If the first variable +! is a constant, then rss(1) is the sum of squares of +! (y - ybar) where ybar is the average value of y. +! sserr = residual sum of squares with all of the variables included. +! row_ptr() = array of indices of first elements in each row of R. +! +!-------------------------------------------------------------------------- + +! General declarations + +IMPLICIT NONE + +INTEGER, SAVE :: nobs, ncol, r_dim +INTEGER, ALLOCATABLE, SAVE :: vorder(:), row_ptr(:) +LOGICAL, SAVE :: initialized = .false., & + tol_set = .false., rss_set = .false. + +! Note. dp is being set to give at least 12 decimal digit +! representation of floating point numbers. This should be adequate +! for most problems except the fitting of polynomials. dp is +! being set so that the same code can be run on PCs and Unix systems, +! which will usually represent floating-point numbers in `double +! precision', and other systems with larger word lengths which will +! give similar accuracy in `single precision'. + +INTEGER, PARAMETER :: dp = SELECTED_REAL_KIND(12,60) +REAL (dp), ALLOCATABLE, SAVE :: d(:), rhs(:), r(:), tol(:), rss(:) +REAL (dp), SAVE :: zero = 0.0_dp, one = 1.0_dp, vsmall +REAL (dp), SAVE :: sserr, toly + +PUBLIC :: dp, nobs, ncol, r_dim, vorder, row_ptr, & + initialized, tol_set, rss_set, & + d, rhs, r, tol, rss, sserr +PRIVATE :: zero, one, vsmall + + +CONTAINS + +SUBROUTINE startup(nvar, fit_const) + +! Allocates dimensions for arrays and initializes to zero +! The calling program must set nvar = the number of variables, and +! fit_const = .true. if a constant is to be included in the model, +! otherwise fit_const = .false. +! +!-------------------------------------------------------------------------- + +IMPLICIT NONE +INTEGER, INTENT(IN) :: nvar +LOGICAL, INTENT(IN) :: fit_const + +! Local variable +INTEGER :: i + +vsmall = 10. * TINY(zero) + +nobs = 0 +IF (fit_const) THEN + ncol = nvar + 1 +ELSE + ncol = nvar +END IF + +IF (initialized) DEALLOCATE(d, rhs, r, tol, rss, vorder, row_ptr) +r_dim = ncol * (ncol - 1)/2 +ALLOCATE( d(ncol), rhs(ncol), r(r_dim), tol(ncol), rss(ncol), vorder(ncol), & + row_ptr(ncol) ) + +d = zero +rhs = zero +r = zero +sserr = zero + +IF (fit_const) THEN + DO i = 1, ncol + vorder(i) = i-1 + END DO +ELSE + DO i = 1, ncol + vorder(i) = i + END DO +END IF ! (fit_const) + +! row_ptr(i) is the position of element R(i,i+1) in array r(). + +row_ptr(1) = 1 +DO i = 2, ncol-1 + row_ptr(i) = row_ptr(i-1) + ncol - i + 1 +END DO +row_ptr(ncol) = 0 + +initialized = .true. +tol_set = .false. +rss_set = .false. + +RETURN +END SUBROUTINE startup + + + + +SUBROUTINE includ(weight, xrow, yelem) + +! ALGORITHM AS75.1 APPL. STATIST. (1974) VOL.23, NO. 3 + +! Calling this routine updates D, R, RHS and SSERR by the +! inclusion of xrow, yelem with the specified weight. + +! *** WARNING Array XROW is overwritten. + +! N.B. As this routine will be called many times in most applications, +! checks have been eliminated. +! +!-------------------------------------------------------------------------- + + +IMPLICIT NONE +REAL (dp),INTENT(IN) :: weight, yelem +REAL (dp), DIMENSION(:), INTENT(IN OUT) :: xrow + +! Local variables + +INTEGER :: i, k, nextr +REAL (dp) :: w, y, xi, di, wxi, dpi, cbar, sbar, xk + +nobs = nobs + 1 +w = weight +y = yelem +rss_set = .false. +nextr = 1 +DO i = 1, ncol + +! Skip unnecessary transformations. Test on exact zeroes must be +! used or stability can be destroyed. + + IF (ABS(w) < vsmall) RETURN + xi = xrow(i) + IF (ABS(xi) < vsmall) THEN + nextr = nextr + ncol - i + ELSE + di = d(i) + wxi = w * xi + dpi = di + wxi*xi + cbar = di / dpi + sbar = wxi / dpi + w = cbar * w + d(i) = dpi + DO k = i+1, ncol + xk = xrow(k) + xrow(k) = xk - xi * r(nextr) + r(nextr) = cbar * r(nextr) + sbar * xk + nextr = nextr + 1 + END DO + xk = y + y = xk - xi * rhs(i) + rhs(i) = cbar * rhs(i) + sbar * xk + END IF +END DO ! i = 1, ncol + +! Y * SQRT(W) is now equal to the Brown, Durbin & Evans recursive +! residual. + +sserr = sserr + w * y * y + +RETURN +END SUBROUTINE includ + + + +SUBROUTINE regcf(beta, nreq, ifault) + +! ALGORITHM AS274 APPL. STATIST. (1992) VOL.41, NO. 2 + +! Modified version of AS75.4 to calculate regression coefficients +! for the first NREQ variables, given an orthogonal reduction from +! AS75.1. +! +!-------------------------------------------------------------------------- + +IMPLICIT NONE +INTEGER, INTENT(IN) :: nreq +INTEGER, INTENT(OUT) :: ifault +REAL (dp), DIMENSION(:), INTENT(OUT) :: beta + +! Local variables + +INTEGER :: i, j, nextr + +! Some checks. + +ifault = 0 +IF (nreq < 1 .OR. nreq > ncol) ifault = ifault + 4 +IF (ifault /= 0) RETURN + +IF (.NOT. tol_set) CALL tolset() + +DO i = nreq, 1, -1 + IF (SQRT(d(i)) < tol(i)) THEN + beta(i) = zero + d(i) = zero + ifault = -i + ELSE + beta(i) = rhs(i) + nextr = row_ptr(i) + DO j = i+1, nreq + beta(i) = beta(i) - r(nextr) * beta(j) + nextr = nextr + 1 + END DO ! j = i+1, nreq + END IF +END DO ! i = nreq, 1, -1 + +RETURN +END SUBROUTINE regcf + + + +SUBROUTINE tolset(eps) + +! ALGORITHM AS274 APPL. STATIST. (1992) VOL.41, NO. 2 + +! Sets up array TOL for testing for zeroes in an orthogonal +! reduction formed using AS75.1. + +REAL (dp), INTENT(IN), OPTIONAL :: eps + +! Unless the argument eps is set, it is assumed that the input data are +! recorded to full machine accuracy. This is often not the case. +! If, for instance, the data are recorded to `single precision' of about +! 6-7 significant decimal digits, then singularities will not be detected. +! It is suggested that in this case eps should be set equal to +! 10.0 * EPSILON(1.0) +! If the data are recorded to say 4 significant decimals, then eps should +! be set to 1.0E-03 +! The above comments apply to the predictor variables, not to the +! dependent variable. + +! Correction - 19 August 2002 +! When negative weights are used, it is possible for an alement of D +! to be negative. + +! Local variables. +! +!-------------------------------------------------------------------------- + +! Local variables + +INTEGER :: col, row, pos +REAL (dp) :: eps1, ten = 10.0, total, work(ncol) + +! EPS is a machine-dependent constant. + +IF (PRESENT(eps)) THEN + eps1 = MAX(ABS(eps), ten * EPSILON(ten)) +ELSE + eps1 = ten * EPSILON(ten) +END IF + +! Set tol(i) = sum of absolute values in column I of R after +! scaling each element by the square root of its row multiplier, +! multiplied by EPS1. + +work = SQRT(ABS(d)) +DO col = 1, ncol + pos = col - 1 + total = work(col) + DO row = 1, col-1 + total = total + ABS(r(pos)) * work(row) + pos = pos + ncol - row - 1 + END DO + tol(col) = eps1 * total +END DO + +tol_set = .TRUE. +RETURN +END SUBROUTINE tolset + + + + +SUBROUTINE sing(lindep, ifault) + +! ALGORITHM AS274 APPL. STATIST. (1992) VOL.41, NO. 2 + +! Checks for singularities, reports, and adjusts orthogonal +! reductions produced by AS75.1. + +! Correction - 19 August 2002 +! When negative weights are used, it is possible for an alement of D +! to be negative. + +! Auxiliary routines called: INCLUD, TOLSET +! +!-------------------------------------------------------------------------- + +INTEGER, INTENT(OUT) :: ifault +LOGICAL, DIMENSION(:), INTENT(OUT) :: lindep + +! Local variables + +REAL (dp) :: temp, x(ncol), work(ncol), y, weight +INTEGER :: pos, row, pos2 + +ifault = 0 + +work = SQRT(ABS(d)) +IF (.NOT. tol_set) CALL tolset() + +DO row = 1, ncol + temp = tol(row) + pos = row_ptr(row) ! pos = location of first element in row + +! If diagonal element is near zero, set it to zero, set appropriate +! element of LINDEP, and use INCLUD to augment the projections in +! the lower rows of the orthogonalization. + + lindep(row) = .FALSE. + IF (work(row) <= temp) THEN + lindep(row) = .TRUE. + ifault = ifault - 1 + IF (row < ncol) THEN + pos2 = pos + ncol - row - 1 + x = zero + x(row+1:ncol) = r(pos:pos2) + y = rhs(row) + weight = d(row) + r(pos:pos2) = zero + d(row) = zero + rhs(row) = zero + CALL includ(weight, x, y) + ! INCLUD automatically increases the number + ! of cases each time it is called. + nobs = nobs - 1 + ELSE + sserr = sserr + d(row) * rhs(row)**2 + END IF ! (row < ncol) + END IF ! (work(row) <= temp) +END DO ! row = 1, ncol + +RETURN +END SUBROUTINE sing + + + +SUBROUTINE ss() + +! ALGORITHM AS274 APPL. STATIST. (1992) VOL.41, NO. 2 + +! Calculates partial residual sums of squares from an orthogonal +! reduction from AS75.1. +! +!-------------------------------------------------------------------------- + +! Local variables + +INTEGER :: i +REAL (dp) :: total + +total = sserr +rss(ncol) = sserr +DO i = ncol, 2, -1 + total = total + d(i) * rhs(i)**2 + rss(i-1) = total +END DO + +rss_set = .TRUE. +RETURN +END SUBROUTINE ss + + + +SUBROUTINE cov(nreq, var, covmat, dimcov, sterr, ifault) + +! ALGORITHM AS274 APPL. STATIST. (1992) VOL.41, NO. 2 + +! Calculate covariance matrix for regression coefficients for the +! first nreq variables, from an orthogonal reduction produced from +! AS75.1. + +! Auxiliary routine called: INV +! +!-------------------------------------------------------------------------- + +INTEGER, INTENT(IN) :: nreq, dimcov +INTEGER, INTENT(OUT) :: ifault +REAL (dp), INTENT(OUT) :: var +REAL (dp), DIMENSION(:), INTENT(OUT) :: covmat, sterr + +! Local variables. + +INTEGER :: dim_rinv, pos, row, start, pos2, col, pos1, k +REAL (dp) :: total +REAL (dp), ALLOCATABLE :: rinv(:) + +! Check that dimension of array covmat is adequate. + +IF (dimcov < nreq*(nreq+1)/2) THEN + ifault = 1 + RETURN +END IF + +! Check for small or zero multipliers on the diagonal. + +ifault = 0 +DO row = 1, nreq + IF (ABS(d(row)) < vsmall) ifault = -row +END DO +IF (ifault /= 0) RETURN + +! Calculate estimate of the residual variance. + +IF (nobs > nreq) THEN + IF (.NOT. rss_set) CALL ss() + var = rss(nreq) / (nobs - nreq) +ELSE + ifault = 2 + RETURN +END IF + +dim_rinv = nreq*(nreq-1)/2 +ALLOCATE ( rinv(dim_rinv) ) + +CALL INV(nreq, rinv) +pos = 1 +start = 1 +DO row = 1, nreq + pos2 = start + DO col = row, nreq + pos1 = start + col - row + IF (row == col) THEN + total = one / d(col) + ELSE + total = rinv(pos1-1) / d(col) + END IF + DO K = col+1, nreq + total = total + rinv(pos1) * rinv(pos2) / d(k) + pos1 = pos1 + 1 + pos2 = pos2 + 1 + END DO ! K = col+1, nreq + covmat(pos) = total * var + IF (row == col) sterr(row) = SQRT(covmat(pos)) + pos = pos + 1 + END DO ! col = row, nreq + start = start + nreq - row +END DO ! row = 1, nreq + +DEALLOCATE(rinv) +RETURN +END SUBROUTINE cov + + + +SUBROUTINE inv(nreq, rinv) + +! ALGORITHM AS274 APPL. STATIST. (1992) VOL.41, NO. 2 + +! Invert first nreq rows and columns of Cholesky factorization +! produced by AS 75.1. +! +!-------------------------------------------------------------------------- + +INTEGER, INTENT(IN) :: nreq +REAL (dp), DIMENSION(:), INTENT(OUT) :: rinv + +! Local variables. + +INTEGER :: pos, row, col, start, k, pos1, pos2 +REAL (dp) :: total + +! Invert R ignoring row multipliers, from the bottom up. + +pos = nreq * (nreq-1)/2 +DO row = nreq-1, 1, -1 + start = row_ptr(row) + DO col = nreq, row+1, -1 + pos1 = start + pos2 = pos + total = zero + DO k = row+1, col-1 + pos2 = pos2 + nreq - k + total = total - r(pos1) * rinv(pos2) + pos1 = pos1 + 1 + END DO ! k = row+1, col-1 + rinv(pos) = total - r(pos1) + pos = pos - 1 + END DO ! col = nreq, row+1, -1 +END DO ! row = nreq-1, 1, -1 + +RETURN +END SUBROUTINE inv + + + +SUBROUTINE partial_corr(in, cormat, dimc, ycorr, ifault) + +! Replaces subroutines PCORR and COR of: +! ALGORITHM AS274 APPL. STATIST. (1992) VOL.41, NO. 2 + +! Calculate partial correlations after the variables in rows +! 1, 2, ..., IN have been forced into the regression. +! If IN = 1, and the first row of R represents a constant in the +! model, then the usual simple correlations are returned. + +! If IN = 0, the value returned in array CORMAT for the correlation +! of variables Xi & Xj is: +! sum ( Xi.Xj ) / Sqrt ( sum (Xi^2) . sum (Xj^2) ) + +! On return, array CORMAT contains the upper triangle of the matrix of +! partial correlations stored by rows, excluding the 1's on the diagonal. +! e.g. if IN = 2, the consecutive elements returned are: +! (3,4) (3,5) ... (3,ncol), (4,5) (4,6) ... (4,ncol), etc. +! Array YCORR stores the partial correlations with the Y-variable +! starting with YCORR(IN+1) = partial correlation with the variable in +! position (IN+1). +! +!-------------------------------------------------------------------------- + +INTEGER, INTENT(IN) :: in, dimc +INTEGER, INTENT(OUT) :: ifault +REAL (dp), DIMENSION(:), INTENT(OUT) :: cormat, ycorr + +! Local variables. + +INTEGER :: base_pos, pos, row, col, col1, col2, pos1, pos2 +REAL (dp) :: rms(in+1:ncol), sumxx, sumxy, sumyy, work(in+1:ncol) + +! Some checks. + +ifault = 0 +IF (in < 0 .OR. in > ncol-1) ifault = ifault + 4 +IF (dimc < (ncol-in)*(ncol-in-1)/2) ifault = ifault + 8 +IF (ifault /= 0) RETURN + +! Base position for calculating positions of elements in row (IN+1) of R. + +base_pos = in*ncol - (in+1)*(in+2)/2 + +! Calculate 1/RMS of elements in columns from IN to (ncol-1). + +IF (d(in+1) > zero) rms(in+1) = one / SQRT(d(in+1)) +DO col = in+2, ncol + pos = base_pos + col + sumxx = d(col) + DO row = in+1, col-1 + sumxx = sumxx + d(row) * r(pos)**2 + pos = pos + ncol - row - 1 + END DO ! row = in+1, col-1 + IF (sumxx > zero) THEN + rms(col) = one / SQRT(sumxx) + ELSE + rms(col) = zero + ifault = -col + END IF ! (sumxx > zero) +END DO ! col = in+1, ncol-1 + +! Calculate 1/RMS for the Y-variable + +sumyy = sserr +DO row = in+1, ncol + sumyy = sumyy + d(row) * rhs(row)**2 +END DO ! row = in+1, ncol +IF (sumyy > zero) sumyy = one / SQRT(sumyy) + +! Calculate sums of cross-products. +! These are obtained by taking dot products of pairs of columns of R, +! but with the product for each row multiplied by the row multiplier +! in array D. + +pos = 1 +DO col1 = in+1, ncol + sumxy = zero + work(col1+1:ncol) = zero + pos1 = base_pos + col1 + DO row = in+1, col1-1 + pos2 = pos1 + 1 + DO col2 = col1+1, ncol + work(col2) = work(col2) + d(row) * r(pos1) * r(pos2) + pos2 = pos2 + 1 + END DO ! col2 = col1+1, ncol + sumxy = sumxy + d(row) * r(pos1) * rhs(row) + pos1 = pos1 + ncol - row - 1 + END DO ! row = in+1, col1-1 + +! Row COL1 has an implicit 1 as its first element (in column COL1) + + pos2 = pos1 + 1 + DO col2 = col1+1, ncol + work(col2) = work(col2) + d(col1) * r(pos2) + pos2 = pos2 + 1 + cormat(pos) = work(col2) * rms(col1) * rms(col2) + pos = pos + 1 + END DO ! col2 = col1+1, ncol + sumxy = sumxy + d(col1) * rhs(col1) + ycorr(col1) = sumxy * rms(col1) * sumyy +END DO ! col1 = in+1, ncol-1 + +ycorr(1:in) = zero + +RETURN +END SUBROUTINE partial_corr + + + + +SUBROUTINE vmove(from, to, ifault) + +! ALGORITHM AS274 APPL. STATIST. (1992) VOL.41, NO. 2 + +! Move variable from position FROM to position TO in an +! orthogonal reduction produced by AS75.1. +! +!-------------------------------------------------------------------------- + +INTEGER, INTENT(IN) :: from, to +INTEGER, INTENT(OUT) :: ifault + +! Local variables + +REAL (dp) :: d1, d2, x, d1new, d2new, cbar, sbar, y +INTEGER :: m, first, last, inc, m1, m2, mp1, col, pos, row + +! Check input parameters + +ifault = 0 +IF (from < 1 .OR. from > ncol) ifault = ifault + 4 +IF (to < 1 .OR. to > ncol) ifault = ifault + 8 +IF (ifault /= 0) RETURN + +IF (from == to) RETURN + +IF (.NOT. rss_set) CALL ss() + +IF (from < to) THEN + first = from + last = to - 1 + inc = 1 +ELSE + first = from - 1 + last = to + inc = -1 +END IF + +DO m = first, last, inc + +! Find addresses of first elements of R in rows M and (M+1). + + m1 = row_ptr(m) + m2 = row_ptr(m+1) + mp1 = m + 1 + d1 = d(m) + d2 = d(mp1) + +! Special cases. + + IF (d1 < vsmall .AND. d2 < vsmall) GO TO 40 + x = r(m1) + IF (ABS(x) * SQRT(d1) < tol(mp1)) THEN + x = zero + END IF + IF (d1 < vsmall .OR. ABS(x) < vsmall) THEN + d(m) = d2 + d(mp1) = d1 + r(m1) = zero + DO col = m+2, ncol + m1 = m1 + 1 + x = r(m1) + r(m1) = r(m2) + r(m2) = x + m2 = m2 + 1 + END DO ! col = m+2, ncol + x = rhs(m) + rhs(m) = rhs(mp1) + rhs(mp1) = x + GO TO 40 + ELSE IF (d2 < vsmall) THEN + d(m) = d1 * x**2 + r(m1) = one / x + r(m1+1:m1+ncol-m-1) = r(m1+1:m1+ncol-m-1) / x + rhs(m) = rhs(m) / x + GO TO 40 + END IF + +! Planar rotation in regular case. + + d1new = d2 + d1*x**2 + cbar = d2 / d1new + sbar = x * d1 / d1new + d2new = d1 * cbar + d(m) = d1new + d(mp1) = d2new + r(m1) = sbar + DO col = m+2, ncol + m1 = m1 + 1 + y = r(m1) + r(m1) = cbar*r(m2) + sbar*y + r(m2) = y - x*r(m2) + m2 = m2 + 1 + END DO ! col = m+2, ncol + y = rhs(m) + rhs(m) = cbar*rhs(mp1) + sbar*y + rhs(mp1) = y - x*rhs(mp1) + +! Swap columns M and (M+1) down to row (M-1). + + 40 pos = m + DO row = 1, m-1 + x = r(pos) + r(pos) = r(pos-1) + r(pos-1) = x + pos = pos + ncol - row - 1 + END DO ! row = 1, m-1 + +! Adjust variable order (VORDER), the tolerances (TOL) and +! the vector of residual sums of squares (RSS). + + m1 = vorder(m) + vorder(m) = vorder(mp1) + vorder(mp1) = m1 + x = tol(m) + tol(m) = tol(mp1) + tol(mp1) = x + rss(m) = rss(mp1) + d(mp1) * rhs(mp1)**2 +END DO + +RETURN +END SUBROUTINE vmove + + + +SUBROUTINE reordr(list, n, pos1, ifault) + +! ALGORITHM AS274 APPL. STATIST. (1992) VOL.41, NO. 2 + +! Re-order the variables in an orthogonal reduction produced by +! AS75.1 so that the N variables in LIST start at position POS1, +! though will not necessarily be in the same order as in LIST. +! Any variables in VORDER before position POS1 are not moved. + +! Auxiliary routine called: VMOVE +! +!-------------------------------------------------------------------------- + +INTEGER, INTENT(IN) :: n, pos1 +INTEGER, DIMENSION(:), INTENT(IN) :: list +INTEGER, INTENT(OUT) :: ifault + +! Local variables. + +INTEGER :: next, i, l, j + +! Check N. + +ifault = 0 +IF (n < 1 .OR. n > ncol+1-pos1) ifault = ifault + 4 +IF (ifault /= 0) RETURN + +! Work through VORDER finding variables which are in LIST. + +next = pos1 +i = pos1 +10 l = vorder(i) +DO j = 1, n + IF (l == list(j)) GO TO 40 +END DO +30 i = i + 1 +IF (i <= ncol) GO TO 10 + +! If this point is reached, one or more variables in LIST has not +! been found. + +ifault = 8 +RETURN + +! Variable L is in LIST; move it up to position NEXT if it is not +! already there. + +40 IF (i > next) CALL vmove(i, next, ifault) +next = next + 1 +IF (next < n+pos1) GO TO 30 + +RETURN +END SUBROUTINE reordr + + + +SUBROUTINE hdiag(xrow, nreq, hii, ifault) + +! ALGORITHM AS274 APPL. STATIST. (1992) VOL.41, NO. 2 +! +! -1 -1 +! The hat matrix H = x(X'X) x' = x(R'DR) x' = z'Dz + +! -1 +! where z = x'R + +! Here we only calculate the diagonal element hii corresponding to one +! row (xrow). The variance of the i-th least-squares residual is (1 - hii). +!-------------------------------------------------------------------------- + +INTEGER, INTENT(IN) :: nreq +INTEGER, INTENT(OUT) :: ifault +REAL (dp), DIMENSION(:), INTENT(IN) :: xrow +REAL (dp), INTENT(OUT) :: hii + +! Local variables + +INTEGER :: col, row, pos +REAL (dp) :: total, wk(ncol) + +! Some checks + +ifault = 0 +IF (nreq > ncol) ifault = ifault + 4 +IF (ifault /= 0) RETURN + +! The elements of xrow.inv(R).sqrt(D) are calculated and stored in WK. + +hii = zero +DO col = 1, nreq + IF (SQRT(d(col)) <= tol(col)) THEN + wk(col) = zero + ELSE + pos = col - 1 + total = xrow(col) + DO row = 1, col-1 + total = total - wk(row)*r(pos) + pos = pos + ncol - row - 1 + END DO ! row = 1, col-1 + wk(col) = total + hii = hii + total**2 / d(col) + END IF +END DO ! col = 1, nreq + +RETURN +END SUBROUTINE hdiag + + + +FUNCTION varprd(x, nreq) RESULT(fn_val) + +! Calculate the variance of x'b where b consists of the first nreq +! least-squares regression coefficients. +! +!-------------------------------------------------------------------------- + +INTEGER, INTENT(IN) :: nreq +REAL (dp), DIMENSION(:), INTENT(IN) :: x +REAL (dp) :: fn_val + +! Local variables + +INTEGER :: ifault, row +REAL (dp) :: var, wk(nreq) + +! Check input parameter values + +fn_val = zero +ifault = 0 +IF (nreq < 1 .OR. nreq > ncol) ifault = ifault + 4 +IF (nobs <= nreq) ifault = ifault + 8 +IF (ifault /= 0) THEN + WRITE(*, '(1x, a, i4)') 'Error in function VARPRD: ifault =', ifault + RETURN +END IF + +! Calculate the residual variance estimate. + +var = sserr / (nobs - nreq) + +! Variance of x'b = var.x'(inv R)(inv D)(inv R')x +! First call BKSUB2 to calculate (inv R')x by back-substitution. + +CALL BKSUB2(x, wk, nreq) +DO row = 1, nreq + IF(d(row) > tol(row)) fn_val = fn_val + wk(row)**2 / d(row) +END DO + +fn_val = fn_val * var + +RETURN +END FUNCTION varprd + + + +SUBROUTINE bksub2(x, b, nreq) + +! Solve x = R'b for b given x, using only the first nreq rows and +! columns of R, and only the first nreq elements of R. +! +!-------------------------------------------------------------------------- + +INTEGER, INTENT(IN) :: nreq +REAL (dp), DIMENSION(:), INTENT(IN) :: x +REAL (dp), DIMENSION(:), INTENT(OUT) :: b + +! Local variables + +INTEGER :: pos, row, col +REAL (dp) :: temp + +! Solve by back-substitution, starting from the top. + +DO row = 1, nreq + pos = row - 1 + temp = x(row) + DO col = 1, row-1 + temp = temp - r(pos)*b(col) + pos = pos + ncol - col - 1 + END DO + b(row) = temp +END DO + +RETURN +END SUBROUTINE bksub2 + + +END MODULE lsq diff --git a/kp_scripts/util/lsq.mod b/kp_scripts/util/lsq.mod new file mode 100644 index 0000000..319a21f --- /dev/null +++ b/kp_scripts/util/lsq.mod @@ -0,0 +1,207 @@ +GFORTRAN module version '10' created from lsq.f90 +MD5:e2c0d7d5e3709bd95ab57a404d49dd51 -- If you edit this, you'll get what you deserve. + +(() () () () () () () () () () () () () () () () () () () () () () () () +() () ()) + +() + +() + +() + +() + +() + +(2 'bksub2' 'lsq' '' 1 ((PROCEDURE UNKNOWN-INTENT MODULE-PROC DECL +UNKNOWN 0 0 SUBROUTINE IMPLICIT_PURE ALWAYS_EXPLICIT) (UNKNOWN 0 0 0 0 +UNKNOWN ()) 3 0 (4 5 6) () 0 () () () 0 0) +7 'cov' 'lsq' '' 1 ((PROCEDURE UNKNOWN-INTENT MODULE-PROC DECL UNKNOWN 0 +0 SUBROUTINE ALWAYS_EXPLICIT) (UNKNOWN 0 0 0 0 UNKNOWN ()) 8 0 (9 10 11 +12 13 14) () 0 () () () 0 0) +15 'd' 'lsq' '' 1 ((VARIABLE UNKNOWN-INTENT UNKNOWN-PROC UNKNOWN +EXPLICIT-SAVE 0 0 ALLOCATABLE DIMENSION) (REAL 8 0 0 0 REAL ()) 0 0 () ( +1 0 DEFERRED () ()) 0 () () () 0 0) +16 'dp' 'lsq' '' 1 ((PARAMETER UNKNOWN-INTENT UNKNOWN-PROC UNKNOWN +IMPLICIT-SAVE 0 0) (INTEGER 4 0 0 0 INTEGER ()) 0 0 () (CONSTANT ( +INTEGER 4 0 0 0 INTEGER ()) 0 '8') () 0 () () () 0 0) +17 'hdiag' 'lsq' '' 1 ((PROCEDURE UNKNOWN-INTENT MODULE-PROC DECL +UNKNOWN 0 0 SUBROUTINE IMPLICIT_PURE ALWAYS_EXPLICIT) (UNKNOWN 0 0 0 0 +UNKNOWN ()) 18 0 (19 20 21 22) () 0 () () () 0 0) +23 'includ' 'lsq' '' 1 ((PROCEDURE UNKNOWN-INTENT MODULE-PROC DECL +UNKNOWN 0 0 SUBROUTINE ALWAYS_EXPLICIT) (UNKNOWN 0 0 0 0 UNKNOWN ()) 24 +0 (25 26 27) () 0 () () () 0 0) +28 'initialized' 'lsq' '' 1 ((VARIABLE UNKNOWN-INTENT UNKNOWN-PROC +UNKNOWN EXPLICIT-SAVE 0 0) (LOGICAL 4 0 0 0 LOGICAL ()) 0 0 () () 0 () () +() 0 0) +29 'inv' 'lsq' '' 1 ((PROCEDURE UNKNOWN-INTENT MODULE-PROC DECL UNKNOWN +0 0 SUBROUTINE IMPLICIT_PURE ALWAYS_EXPLICIT) (UNKNOWN 0 0 0 0 UNKNOWN ()) +30 0 (31 32) () 0 () () () 0 0) +33 'lsq' 'lsq' '' 1 ((MODULE UNKNOWN-INTENT UNKNOWN-PROC UNKNOWN UNKNOWN +0 0) (UNKNOWN 0 0 0 0 UNKNOWN ()) 0 0 () () 0 () () () 0 0) +34 'ncol' 'lsq' '' 1 ((VARIABLE UNKNOWN-INTENT UNKNOWN-PROC UNKNOWN +EXPLICIT-SAVE 0 0) (INTEGER 4 0 0 0 INTEGER ()) 0 0 () () 0 () () () 0 0) +35 'nobs' 'lsq' '' 1 ((VARIABLE UNKNOWN-INTENT UNKNOWN-PROC UNKNOWN +EXPLICIT-SAVE 0 0) (INTEGER 4 0 0 0 INTEGER ()) 0 0 () () 0 () () () 0 0) +36 'partial_corr' 'lsq' '' 1 ((PROCEDURE UNKNOWN-INTENT MODULE-PROC DECL +UNKNOWN 0 0 SUBROUTINE IMPLICIT_PURE ALWAYS_EXPLICIT) (UNKNOWN 0 0 0 0 +UNKNOWN ()) 37 0 (38 39 40 41 42) () 0 () () () 0 0) +43 'r' 'lsq' '' 1 ((VARIABLE UNKNOWN-INTENT UNKNOWN-PROC UNKNOWN +EXPLICIT-SAVE 0 0 ALLOCATABLE DIMENSION) (REAL 8 0 0 0 REAL ()) 0 0 () ( +1 0 DEFERRED () ()) 0 () () () 0 0) +44 'r_dim' 'lsq' '' 1 ((VARIABLE UNKNOWN-INTENT UNKNOWN-PROC UNKNOWN +EXPLICIT-SAVE 0 0) (INTEGER 4 0 0 0 INTEGER ()) 0 0 () () 0 () () () 0 0) +45 'regcf' 'lsq' '' 1 ((PROCEDURE UNKNOWN-INTENT MODULE-PROC DECL +UNKNOWN 0 0 SUBROUTINE ALWAYS_EXPLICIT) (UNKNOWN 0 0 0 0 UNKNOWN ()) 46 +0 (47 48 49) () 0 () () () 0 0) +50 'reordr' 'lsq' '' 1 ((PROCEDURE UNKNOWN-INTENT MODULE-PROC DECL +UNKNOWN 0 0 SUBROUTINE ALWAYS_EXPLICIT) (UNKNOWN 0 0 0 0 UNKNOWN ()) 51 +0 (52 53 54 55) () 0 () () () 0 0) +56 'rhs' 'lsq' '' 1 ((VARIABLE UNKNOWN-INTENT UNKNOWN-PROC UNKNOWN +EXPLICIT-SAVE 0 0 ALLOCATABLE DIMENSION) (REAL 8 0 0 0 REAL ()) 0 0 () ( +1 0 DEFERRED () ()) 0 () () () 0 0) +57 'row_ptr' 'lsq' '' 1 ((VARIABLE UNKNOWN-INTENT UNKNOWN-PROC UNKNOWN +EXPLICIT-SAVE 0 0 ALLOCATABLE DIMENSION) (INTEGER 4 0 0 0 INTEGER ()) 0 +0 () (1 0 DEFERRED () ()) 0 () () () 0 0) +58 'rss' 'lsq' '' 1 ((VARIABLE UNKNOWN-INTENT UNKNOWN-PROC UNKNOWN +EXPLICIT-SAVE 0 0 ALLOCATABLE DIMENSION) (REAL 8 0 0 0 REAL ()) 0 0 () ( +1 0 DEFERRED () ()) 0 () () () 0 0) +59 'rss_set' 'lsq' '' 1 ((VARIABLE UNKNOWN-INTENT UNKNOWN-PROC UNKNOWN +EXPLICIT-SAVE 0 0) (LOGICAL 4 0 0 0 LOGICAL ()) 0 0 () () 0 () () () 0 0) +60 'selected_real_kind' '(intrinsic)' '' 1 ((PROCEDURE UNKNOWN-INTENT +UNKNOWN-PROC UNKNOWN UNKNOWN 0 0 FUNCTION) (UNKNOWN 0 0 0 0 UNKNOWN ()) +0 0 () () 60 () () () 0 0) +61 'sing' 'lsq' '' 1 ((PROCEDURE UNKNOWN-INTENT MODULE-PROC DECL UNKNOWN +0 0 SUBROUTINE ALWAYS_EXPLICIT) (UNKNOWN 0 0 0 0 UNKNOWN ()) 62 0 (63 64) +() 0 () () () 0 0) +65 'ss' 'lsq' '' 1 ((PROCEDURE UNKNOWN-INTENT MODULE-PROC DECL UNKNOWN 0 +0 SUBROUTINE) (UNKNOWN 0 0 0 0 UNKNOWN ()) 0 0 () () 0 () () () 0 0) +66 'sserr' 'lsq' '' 1 ((VARIABLE UNKNOWN-INTENT UNKNOWN-PROC UNKNOWN +EXPLICIT-SAVE 0 0) (REAL 8 0 0 0 REAL ()) 0 0 () () 0 () () () 0 0) +67 'startup' 'lsq' '' 1 ((PROCEDURE UNKNOWN-INTENT MODULE-PROC DECL +UNKNOWN 0 0 SUBROUTINE) (UNKNOWN 0 0 0 0 UNKNOWN ()) 68 0 (69 70) () 0 () +() () 0 0) +71 'tol' 'lsq' '' 1 ((VARIABLE UNKNOWN-INTENT UNKNOWN-PROC UNKNOWN +EXPLICIT-SAVE 0 0 ALLOCATABLE DIMENSION) (REAL 8 0 0 0 REAL ()) 0 0 () ( +1 0 DEFERRED () ()) 0 () () () 0 0) +72 'tol_set' 'lsq' '' 1 ((VARIABLE UNKNOWN-INTENT UNKNOWN-PROC UNKNOWN +EXPLICIT-SAVE 0 0) (LOGICAL 4 0 0 0 LOGICAL ()) 0 0 () () 0 () () () 0 0) +73 'tolset' 'lsq' '' 1 ((PROCEDURE UNKNOWN-INTENT MODULE-PROC DECL +UNKNOWN 0 0 SUBROUTINE ALWAYS_EXPLICIT) (UNKNOWN 0 0 0 0 UNKNOWN ()) 74 +0 (75) () 0 () () () 0 0) +76 'toly' 'lsq' '' 1 ((VARIABLE UNKNOWN-INTENT UNKNOWN-PROC UNKNOWN +EXPLICIT-SAVE 0 0) (REAL 8 0 0 0 REAL ()) 0 0 () () 0 () () () 0 0) +77 'varprd' 'lsq' '' 1 ((PROCEDURE UNKNOWN-INTENT MODULE-PROC DECL +UNKNOWN 0 0 FUNCTION ALWAYS_EXPLICIT) (REAL 8 0 0 0 REAL ()) 78 0 (79 80) +() 81 () () () 0 0) +82 'vmove' 'lsq' '' 1 ((PROCEDURE UNKNOWN-INTENT MODULE-PROC DECL +UNKNOWN 0 0 SUBROUTINE) (UNKNOWN 0 0 0 0 UNKNOWN ()) 83 0 (84 85 86) () +0 () () () 0 0) +87 'vorder' 'lsq' '' 1 ((VARIABLE UNKNOWN-INTENT UNKNOWN-PROC UNKNOWN +EXPLICIT-SAVE 0 0 ALLOCATABLE DIMENSION) (INTEGER 4 0 0 0 INTEGER ()) 0 +0 () (1 0 DEFERRED () ()) 0 () () () 0 0) +4 'x' '' '' 3 ((VARIABLE IN UNKNOWN-PROC UNKNOWN UNKNOWN 0 0 DIMENSION +DUMMY) (REAL 8 0 0 0 REAL ()) 0 0 () (1 0 ASSUMED_SHAPE (CONSTANT ( +INTEGER 4 0 0 0 INTEGER ()) 0 '1') ()) 0 () () () 0 0) +5 'b' '' '' 3 ((VARIABLE OUT UNKNOWN-PROC UNKNOWN UNKNOWN 0 0 DIMENSION +DUMMY) (REAL 8 0 0 0 REAL ()) 0 0 () (1 0 ASSUMED_SHAPE (CONSTANT ( +INTEGER 4 0 0 0 INTEGER ()) 0 '1') ()) 0 () () () 0 0) +6 'nreq' '' '' 3 ((VARIABLE IN UNKNOWN-PROC UNKNOWN UNKNOWN 0 0 DUMMY) ( +INTEGER 4 0 0 0 INTEGER ()) 0 0 () () 0 () () () 0 0) +9 'nreq' '' '' 8 ((VARIABLE IN UNKNOWN-PROC UNKNOWN UNKNOWN 0 0 DUMMY) ( +INTEGER 4 0 0 0 INTEGER ()) 0 0 () () 0 () () () 0 0) +10 'var' '' '' 8 ((VARIABLE OUT UNKNOWN-PROC UNKNOWN UNKNOWN 0 0 DUMMY) +(REAL 8 0 0 0 REAL ()) 0 0 () () 0 () () () 0 0) +11 'covmat' '' '' 8 ((VARIABLE OUT UNKNOWN-PROC UNKNOWN UNKNOWN 0 0 +DIMENSION DUMMY) (REAL 8 0 0 0 REAL ()) 0 0 () (1 0 ASSUMED_SHAPE ( +CONSTANT (INTEGER 4 0 0 0 INTEGER ()) 0 '1') ()) 0 () () () 0 0) +12 'dimcov' '' '' 8 ((VARIABLE IN UNKNOWN-PROC UNKNOWN UNKNOWN 0 0 DUMMY) +(INTEGER 4 0 0 0 INTEGER ()) 0 0 () () 0 () () () 0 0) +13 'sterr' '' '' 8 ((VARIABLE OUT UNKNOWN-PROC UNKNOWN UNKNOWN 0 0 +DIMENSION DUMMY) (REAL 8 0 0 0 REAL ()) 0 0 () (1 0 ASSUMED_SHAPE ( +CONSTANT (INTEGER 4 0 0 0 INTEGER ()) 0 '1') ()) 0 () () () 0 0) +14 'ifault' '' '' 8 ((VARIABLE OUT UNKNOWN-PROC UNKNOWN UNKNOWN 0 0 +DUMMY) (INTEGER 4 0 0 0 INTEGER ()) 0 0 () () 0 () () () 0 0) +19 'xrow' '' '' 18 ((VARIABLE IN UNKNOWN-PROC UNKNOWN UNKNOWN 0 0 +DIMENSION DUMMY) (REAL 8 0 0 0 REAL ()) 0 0 () (1 0 ASSUMED_SHAPE ( +CONSTANT (INTEGER 4 0 0 0 INTEGER ()) 0 '1') ()) 0 () () () 0 0) +20 'nreq' '' '' 18 ((VARIABLE IN UNKNOWN-PROC UNKNOWN UNKNOWN 0 0 DUMMY) +(INTEGER 4 0 0 0 INTEGER ()) 0 0 () () 0 () () () 0 0) +21 'hii' '' '' 18 ((VARIABLE OUT UNKNOWN-PROC UNKNOWN UNKNOWN 0 0 DUMMY) +(REAL 8 0 0 0 REAL ()) 0 0 () () 0 () () () 0 0) +22 'ifault' '' '' 18 ((VARIABLE OUT UNKNOWN-PROC UNKNOWN UNKNOWN 0 0 +DUMMY) (INTEGER 4 0 0 0 INTEGER ()) 0 0 () () 0 () () () 0 0) +25 'weight' '' '' 24 ((VARIABLE IN UNKNOWN-PROC UNKNOWN UNKNOWN 0 0 +DUMMY) (REAL 8 0 0 0 REAL ()) 0 0 () () 0 () () () 0 0) +26 'xrow' '' '' 24 ((VARIABLE INOUT UNKNOWN-PROC UNKNOWN UNKNOWN 0 0 +DIMENSION DUMMY) (REAL 8 0 0 0 REAL ()) 0 0 () (1 0 ASSUMED_SHAPE ( +CONSTANT (INTEGER 4 0 0 0 INTEGER ()) 0 '1') ()) 0 () () () 0 0) +27 'yelem' '' '' 24 ((VARIABLE IN UNKNOWN-PROC UNKNOWN UNKNOWN 0 0 DUMMY) +(REAL 8 0 0 0 REAL ()) 0 0 () () 0 () () () 0 0) +31 'nreq' '' '' 30 ((VARIABLE IN UNKNOWN-PROC UNKNOWN UNKNOWN 0 0 DUMMY) +(INTEGER 4 0 0 0 INTEGER ()) 0 0 () () 0 () () () 0 0) +32 'rinv' '' '' 30 ((VARIABLE OUT UNKNOWN-PROC UNKNOWN UNKNOWN 0 0 +DIMENSION DUMMY) (REAL 8 0 0 0 REAL ()) 0 0 () (1 0 ASSUMED_SHAPE ( +CONSTANT (INTEGER 4 0 0 0 INTEGER ()) 0 '1') ()) 0 () () () 0 0) +38 'in' '' '' 37 ((VARIABLE IN UNKNOWN-PROC UNKNOWN UNKNOWN 0 0 DUMMY) ( +INTEGER 4 0 0 0 INTEGER ()) 0 0 () () 0 () () () 0 0) +39 'cormat' '' '' 37 ((VARIABLE OUT UNKNOWN-PROC UNKNOWN UNKNOWN 0 0 +DIMENSION DUMMY) (REAL 8 0 0 0 REAL ()) 0 0 () (1 0 ASSUMED_SHAPE ( +CONSTANT (INTEGER 4 0 0 0 INTEGER ()) 0 '1') ()) 0 () () () 0 0) +40 'dimc' '' '' 37 ((VARIABLE IN UNKNOWN-PROC UNKNOWN UNKNOWN 0 0 DUMMY) +(INTEGER 4 0 0 0 INTEGER ()) 0 0 () () 0 () () () 0 0) +41 'ycorr' '' '' 37 ((VARIABLE OUT UNKNOWN-PROC UNKNOWN UNKNOWN 0 0 +DIMENSION DUMMY) (REAL 8 0 0 0 REAL ()) 0 0 () (1 0 ASSUMED_SHAPE ( +CONSTANT (INTEGER 4 0 0 0 INTEGER ()) 0 '1') ()) 0 () () () 0 0) +42 'ifault' '' '' 37 ((VARIABLE OUT UNKNOWN-PROC UNKNOWN UNKNOWN 0 0 +DUMMY) (INTEGER 4 0 0 0 INTEGER ()) 0 0 () () 0 () () () 0 0) +47 'beta' '' '' 46 ((VARIABLE OUT UNKNOWN-PROC UNKNOWN UNKNOWN 0 0 +DIMENSION DUMMY) (REAL 8 0 0 0 REAL ()) 0 0 () (1 0 ASSUMED_SHAPE ( +CONSTANT (INTEGER 4 0 0 0 INTEGER ()) 0 '1') ()) 0 () () () 0 0) +48 'nreq' '' '' 46 ((VARIABLE IN UNKNOWN-PROC UNKNOWN UNKNOWN 0 0 DUMMY) +(INTEGER 4 0 0 0 INTEGER ()) 0 0 () () 0 () () () 0 0) +49 'ifault' '' '' 46 ((VARIABLE OUT UNKNOWN-PROC UNKNOWN UNKNOWN 0 0 +DUMMY) (INTEGER 4 0 0 0 INTEGER ()) 0 0 () () 0 () () () 0 0) +52 'list' '' '' 51 ((VARIABLE IN UNKNOWN-PROC UNKNOWN UNKNOWN 0 0 +DIMENSION DUMMY) (INTEGER 4 0 0 0 INTEGER ()) 0 0 () (1 0 ASSUMED_SHAPE +(CONSTANT (INTEGER 4 0 0 0 INTEGER ()) 0 '1') ()) 0 () () () 0 0) +53 'n' '' '' 51 ((VARIABLE IN UNKNOWN-PROC UNKNOWN UNKNOWN 0 0 DUMMY) ( +INTEGER 4 0 0 0 INTEGER ()) 0 0 () () 0 () () () 0 0) +54 'pos1' '' '' 51 ((VARIABLE IN UNKNOWN-PROC UNKNOWN UNKNOWN 0 0 DUMMY) +(INTEGER 4 0 0 0 INTEGER ()) 0 0 () () 0 () () () 0 0) +55 'ifault' '' '' 51 ((VARIABLE OUT UNKNOWN-PROC UNKNOWN UNKNOWN 0 0 +DUMMY) (INTEGER 4 0 0 0 INTEGER ()) 0 0 () () 0 () () () 0 0) +63 'lindep' '' '' 62 ((VARIABLE OUT UNKNOWN-PROC UNKNOWN UNKNOWN 0 0 +DIMENSION DUMMY) (LOGICAL 4 0 0 0 LOGICAL ()) 0 0 () (1 0 ASSUMED_SHAPE +(CONSTANT (INTEGER 4 0 0 0 INTEGER ()) 0 '1') ()) 0 () () () 0 0) +64 'ifault' '' '' 62 ((VARIABLE OUT UNKNOWN-PROC UNKNOWN UNKNOWN 0 0 +DUMMY) (INTEGER 4 0 0 0 INTEGER ()) 0 0 () () 0 () () () 0 0) +69 'nvar' '' '' 68 ((VARIABLE IN UNKNOWN-PROC UNKNOWN UNKNOWN 0 0 DUMMY) +(INTEGER 4 0 0 0 INTEGER ()) 0 0 () () 0 () () () 0 0) +70 'fit_const' '' '' 68 ((VARIABLE IN UNKNOWN-PROC UNKNOWN UNKNOWN 0 0 +DUMMY) (LOGICAL 4 0 0 0 LOGICAL ()) 0 0 () () 0 () () () 0 0) +75 'eps' '' '' 74 ((VARIABLE IN UNKNOWN-PROC UNKNOWN UNKNOWN 0 0 +OPTIONAL DUMMY) (REAL 8 0 0 0 REAL ()) 0 0 () () 0 () () () 0 0) +79 'x' '' '' 78 ((VARIABLE IN UNKNOWN-PROC UNKNOWN UNKNOWN 0 0 DIMENSION +DUMMY) (REAL 8 0 0 0 REAL ()) 0 0 () (1 0 ASSUMED_SHAPE (CONSTANT ( +INTEGER 4 0 0 0 INTEGER ()) 0 '1') ()) 0 () () () 0 0) +80 'nreq' '' '' 78 ((VARIABLE IN UNKNOWN-PROC UNKNOWN UNKNOWN 0 0 DUMMY) +(INTEGER 4 0 0 0 INTEGER ()) 0 0 () () 0 () () () 0 0) +81 'fn_val' '' '' 78 ((VARIABLE UNKNOWN-INTENT UNKNOWN-PROC UNKNOWN +UNKNOWN 0 0 RESULT ALWAYS_EXPLICIT) (REAL 8 0 0 0 REAL ()) 0 0 () () 0 () +() () 0 0) +84 'from' '' '' 83 ((VARIABLE IN UNKNOWN-PROC UNKNOWN UNKNOWN 0 0 DUMMY) +(INTEGER 4 0 0 0 INTEGER ()) 0 0 () () 0 () () () 0 0) +85 'to' '' '' 83 ((VARIABLE IN UNKNOWN-PROC UNKNOWN UNKNOWN 0 0 DUMMY) ( +INTEGER 4 0 0 0 INTEGER ()) 0 0 () () 0 () () () 0 0) +86 'ifault' '' '' 83 ((VARIABLE OUT UNKNOWN-PROC UNKNOWN UNKNOWN 0 0 +DUMMY) (INTEGER 4 0 0 0 INTEGER ()) 0 0 () () 0 () () () 0 0) +) + +('bksub2' 0 2 'cov' 0 7 'd' 0 15 'dp' 0 16 'hdiag' 0 17 'includ' 0 23 +'initialized' 0 28 'inv' 0 29 'lsq' 0 33 'ncol' 0 34 'nobs' 0 35 +'partial_corr' 0 36 'r' 0 43 'r_dim' 0 44 'regcf' 0 45 'reordr' 0 50 'rhs' +0 56 'row_ptr' 0 57 'rss' 0 58 'rss_set' 0 59 'selected_real_kind' 0 60 +'sing' 0 61 'ss' 0 65 'sserr' 0 66 'startup' 0 67 'tol' 0 71 'tol_set' 0 +72 'tolset' 0 73 'toly' 0 76 'varprd' 0 77 'vmove' 0 82 'vorder' 0 87) diff --git a/kp_scripts/util/lsq.o b/kp_scripts/util/lsq.o new file mode 100644 index 0000000..dfed8c7 Binary files /dev/null and b/kp_scripts/util/lsq.o differ diff --git a/kp_scripts/util/make_coverage_masks.py b/kp_scripts/util/make_coverage_masks.py new file mode 100644 index 0000000..5a4c17e --- /dev/null +++ b/kp_scripts/util/make_coverage_masks.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 + +# make_coverage_masks.py -- create masks for SLCs input geolist, to help determine satellite coverage + +import sys +import os + +if len(sys.argv) < 3: + print ('Usage: make_coverage_masks.py geolist nr naz ') + sys.exit(1) + +# Get your arguments from inputs +geolist=sys.argv[1] +nr=sys.argv[2] +naz=sys.argv[3] + +update_flag = 'N' +if len(sys.argv)>3: + update_flag = sys.argv[4] + +# Run through geolist to get the appropriate mask + +fgeos=open(geolist,'r') +geos=fgeos.readlines() +fgeos.close() + +for line in geos: + words=line.split('/') + filename=words[-1].strip() + outfile=filename.replace('geo','mask') + if (os.path.isfile(outfile)==False) or (update_flag=='Y'): + # make the mask + command="$PROC_HOME/kp_scripts/util/coverage_mask "+filename+' '+nr+' '+naz+' '+outfile + print(' '+command) + ret=os.system(command) diff --git a/kp_scripts/util/make_ml_demrsc.py b/kp_scripts/util/make_ml_demrsc.py new file mode 100644 index 0000000..98ab057 --- /dev/null +++ b/kp_scripts/util/make_ml_demrsc.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 + +# make_ml_demrsc - create a new demrsc file based on the input parameters provided +# allow looks +# create the intlist +# don't subset if you don't need to + +import sys +import string +import os +import math + +if len(sys.argv) < 6: + print ('Usage: make_ml_demrsc.py dem_rsc_file xstart ystart xsize ysize ') + sys.exit(1) + +# Default Parameters +xlooks = '1' + +# Get Required Inputs +demrscfile=sys.argv[1] +xstart=sys.argv[2] +ystart=sys.argv[3] +xsize=sys.argv[4] +ysize=sys.argv[5] + +if len(sys.argv) > 6: + xlooks=sys.argv[6] + +ylooks=xlooks +if len(sys.argv) > 7: + ylooks=sys.argv[7] + +##### ----- GENERATE NEW DEM.RSC FILE FOR MULTILOOKED AND/OR EXTRACTED PORTION ----- ##### +# rsc file for extracted portion +frsc=open('dem.rsc','w') + +# dem params +rsc=open(demrscfile,'r') +words=rsc.readline() +demwidth=words.split()[1] +frsc.write(words.replace(demwidth,str(int(int(xsize)/int(xlooks))))) # width +words=rsc.readline() +demlength=words.split()[1] +frsc.write(words.replace(demlength,str(int(int(ysize)/int(ylooks))))) #length +wordsxfirst=rsc.readline() +wordsyfirst=rsc.readline() +wordsxstep=rsc.readline() +demxstep=wordsxstep.split()[1] +wordsystep=rsc.readline() +demystep=wordsystep.split()[1] +xstep=str(float(demxstep)*int(xlooks)) +ystep=str(float(demystep)*int(ylooks)) +demxfirst=wordsxfirst.split()[1] +xfirst=str(float(demxfirst)+(int(xstart)-1)*float(demxstep)) +frsc.write(wordsxfirst.replace(demxfirst,xfirst)) # x_first +demyfirst=wordsyfirst.split()[1] +yfirst=str(float(demyfirst)+(int(ystart)-1)*float(demystep)) +frsc.write(wordsyfirst.replace(demyfirst,yfirst)) # y_first +frsc.write(wordsxstep.replace(demxstep,xstep)) # x_step +frsc.write(wordsystep.replace(demystep,ystep)) # y_step +words=rsc.readline() +frsc.write(words) # x_unit +words=rsc.readline() +frsc.write(words) # y_unit +words=rsc.readline() +frsc.write(words) # z_offset +words=rsc.readline() +frsc.write(words) # z_scale +words=rsc.readline() +frsc.write(words) # projection +words='xstart '+xstart+'\n' +frsc.write(words) +words='ystart '+ystart+'\n' +frsc.write(words) +words='xsize '+xsize+'\n' +frsc.write(words) +words='ysize '+ysize+'\n' +frsc.write(words) +rsc.close() + diff --git a/kp_scripts/util/merge_slcs.py b/kp_scripts/util/merge_slcs.py new file mode 100644 index 0000000..f9f49d0 --- /dev/null +++ b/kp_scripts/util/merge_slcs.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 +# +# merge_slcs.py - merge slcs when same pass has multiple acquisitions +# + +import sys +import os +import subprocess + +if len(sys.argv) < 2: + print ("Usage: merge_slcs.py path_to_merged ") + sys.exit(0) + +path_to_merged = sys.argv[1] + +# Defaults +update_flag = 'N' + +# Get Optional Inputs +if len(sys.argv)>2: + update_flag = sys.argv[2] + +# size of geofiles +fparams = open('params','r') +demfile = fparams.readline().rstrip() +rscfile = fparams.readline().rstrip() +fparams.close() +fe=open(rscfile,'r') +words=fe.readline() +demwidth=words.split()[1] +words=fe.readline() +demlength=words.split()[1] +fe.close() + +# make a list with all geo slc dates +command = 'ls -1 *.geo' +proc = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True) +(geolist, err) = proc.communicate() +geofiles = geolist.split() +geofiles.sort() # put in alphabetical order + +datelist=[] +dategeo=[] +for i in range(len(geofiles)): + geostring=str(geofiles[i],'UTF-8') + date0=geostring[geostring.find('V_')+2:geostring.find('V_')+10] + pol='V' + if date0[3:5] == 'IW': + pol='H' + dategeo.append(geostring[geostring.find(pol+'_')+2:geostring.find(pol+'_')+10]) + if dategeo[i] in datelist: + print (dategeo[i],' is in datelist') + + if dategeo[i] not in datelist: + print (dategeo[i],' is not in list') + datelist.append(dategeo[i]) + +# create a new list with all geos for a date +if (os.path.isfile('merged_list')) and (update_flag=='N'): + fmerge=open('merged_list','r') + merged=fmerge.readlines() + fmerge.close() + fmerge=open('merged_list','a') +else: + fmerge=open('merged_list','w') + +for i in range(len(datelist)): + indices = [j for j, x in enumerate(dategeo) if x == datelist[i]] + geostringout = path_to_merged+'/'+datelist[i]+'_merged.geo' + orbtimingout = geostringout.replace('.geo','.orbtiming') + if os.path.isdir(path_to_merged)==False: + command='mkdir '+path_to_merged + ret=os.system(command) + + if len(indices) > 1: + check=0 + mergelist = [] + # check to see if all of the files have already been merged + for j in range(len(indices)): + geostring = str(geofiles[indices[j]],'UTF-8') + if geostring+'\n' not in merged: + mergelist.append(geostring) + check=1 + if check==0: + print('Already merged all files for date '+datelist[i]) + continue + + if os.path.isfile(geostringout): + for j in range(len(mergelist)): + geostring = mergelist[j] + command = '$PROC_HOME/util/mergeslcs '+geostringout+' '+geostring+' '+geostringout.replace('.geo','.temp')+' '+demwidth+' '+demlength + print(command) + ret=os.system(command) + # rename the outfile + command = 'mv '+geostringout.replace('.geo','.temp')+' '+geostringout + print(command) + ret=os.system(command) + # make sure we have a matching orbtiming file + if os.path.isfile(orbtimingout)==False: + command = 'cp '+geostring.replace('.geo','.orbtiming')+' '+orbtimingout + print(command) + ret=os.system(command) + fmerge.write(geostring+'\n') + else: + geostring1 = mergelist[0] + for j in range(len(mergelist)-1): + geostring2=mergelist[j+1] + command = '$PROC_HOME/util/mergeslcs '+geostring1+' '+geostring2+' '+geostringout.replace('.geo','.temp')+' '+demwidth+' '+demlength + print(command) + ret=os.system(command) + # rename the outfile + command = 'mv '+geostringout.replace('.geo','.temp')+' '+geostringout + print(command) + ret=os.system(command) + # make sure we have a matching orbtiming file + if os.path.isfile(orbtimingout)==False: + command = 'cp '+geostring.replace('.geo','.orbtiming')+' '+orbtimingout + print(command) + ret=os.system(command) + if j==0: + fmerge.write(geostring1+'\n') + fmerge.write(geostring2+'\n') + + geostring1 = geostringout + else: + if os.path.isfile(geostringout)==False: + geostring = str(geofiles[indices[0]],'UTF-8') + command = 'cp '+geostring+' '+geostringout + print(command) + ret=os.system(command) + fmerge.write(geostring+'\n') + if os.path.isfile(orbtimingout)==False: + command = 'cp '+geostring.replace('.geo','.orbtiming')+' '+orbtimingout + print(command) + ret=os.system(command) + +fmerge.close() +sys.exit(0) diff --git a/kp_scripts/util/multilook_slcs.py b/kp_scripts/util/multilook_slcs.py new file mode 100644 index 0000000..f821550 --- /dev/null +++ b/kp_scripts/util/multilook_slcs.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 + +# multilook_slcs.py - multilook a list of geocoded slcs + +import sys +import os +import sys +import math +import glob + +if len(sys.argv) < 5: + print('Usage: multilook_slcs.py geolist len across-looks down-looks ') + sys.exit(1) + +filelist=sys.argv[1] +filelen=sys.argv[2] +xlooks=sys.argv[3] +ylooks=sys.argv[4] + +update_flag = 'N' +if len(sys.argv)>5: + update_flag = sys.argv[5] + +# make a geolist for the directory, named as the input file +exists = [] +if update_flag == 'N': + exists = glob.glob('*.geo') + +# open the geolist +f=open(filelist,'r') +files=f.readlines() +for line in files: + infile=line.strip() + words = infile.split('/') # only want to geo file name, not its path + outfile=words[-1].replace('.geo','_multi.geo') + if outfile in exists: + print(' Skipping '+outfile+', already exists') + else: + print(' Multilooking '+infile) + ret=os.system('$PROC_HOME/util/yujiepowlooks '+infile+' '+outfile+' '+filelen+' '+xlooks+' '+ylooks) + +sys.exit() + diff --git a/kp_scripts/util/regress_igrams.py b/kp_scripts/util/regress_igrams.py new file mode 100644 index 0000000..5f4e7df --- /dev/null +++ b/kp_scripts/util/regress_igrams.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 + +# regress_igrams - remove elevation-dependent height from unwrapped igrams + +import sys +ver=sys.version_info + +import os +import sys +import math + +if len(sys.argv) < 4: + print ('Usage: regress_igrams.py unwlist demfile len lines location_file ') + sys.exit(1) + +### --- REQUIRED PARAMETERS --- ### +filelist=sys.argv[1] +demfile=sys.argv[2] +width=sys.argv[3] +lines=sys.argv[4] +locationfile=sys.argv[5] + +### --- DEFAULT PARAMETERS --- ### +intmaskflag = 'N' + +### --- CHECK INPUTS --- ### +if len(sys.argv)>6: + intmaskflag = sys.argv[6] + +with open(filelist,'r') as funw: + unwlist=funw.readlines() + +for line in unwlist: + # Get input file name + unwfile = line.rstrip() + + # Get output file name + words=line.split('/') + outfile=words[-1].rstrip() + + if os.path.isfile(outfile)==False: + if intmaskflag == 'Y': + # Cull the input reflocs file based on the interferogram mask, and make a temporary file to load in + maskfile = unwfile.replace('.unw','.mask') + else: + maskfile='' + + command='$PROC_HOME/kp_scripts/util/regressheight '+unwfile+' '+outfile+' '+demfile+' '+locationfile+' '+width+' '+lines + print(' '+command) + ret=os.system(command) + +sys.exit() + diff --git a/kp_scripts/util/regressheight b/kp_scripts/util/regressheight new file mode 100644 index 0000000..0cbd857 Binary files /dev/null and b/kp_scripts/util/regressheight differ diff --git a/kp_scripts/util/regressheight.f90 b/kp_scripts/util/regressheight.f90 new file mode 100644 index 0000000..04b065d --- /dev/null +++ b/kp_scripts/util/regressheight.f90 @@ -0,0 +1,183 @@ +!c regress unwrapped phase vs height to remove stratified atmosphere + + +! Requires lsq* files in same directory +! Compile as: gfortran -o regressheight regressheight.f90 lsq.o + + integer*2, allocatable :: dem(:,:) + real*4, allocatable :: unwrapped(:,:) + integer*4 loc(2,10000000) + integer*4 temploc(2) + integer*8 numloc + real*4 phase(10000000),height(10000000),poly(3) + character*300 ufile,demfile,outfile,locationfile,str,maskfile,intfile + complex, allocatable :: igram(:,:) + integer*1, dimension(:,:),allocatable :: mask + + if(iargc().lt.6)then + print *,'Usage: unwrapped_file outfile demfile locations_file len lines ' + stop + end if + + call getarg(1,ufile) + call getarg(2,outfile) + call getarg(3,demfile) + call getarg(4,locationfile) + call getarg(5,str) + read(str,*)len + call getarg(6,str) + read(str,*)lines + + if(iargc().ge.7)call getarg(7,maskfile) + if(iargc().ge.8)call getarg(8,intfile) + + ! ALLOCATE VARIABLES + allocate (unwrapped(2*len,lines), dem(len,lines)) + allocate(mask(len,lines)) + if (iargc().ge.7)then + ! read in mask file + open(unit=31,file=maskfile,access='stream') + read(31)mask + close(31) + else + mask=-1 + end if + + if(iargc().ge.8)allocate (igram(len,lines)) + +!c get the locations + numloc=0 + open(21,file=locationfile) + do i=1,10000000 + read(21,*,end=99)temploc(1),temploc(2) + if (mask(temploc(1),temploc(2)).ne.0)then + numloc=numloc+1 + loc(1,numloc) = temploc(1) + loc(2,numloc) = temploc(2) + end if +! if(i.lt.10)print *,loc(:,i) + end do +99 nlocs=numloc +! print *,'Locations found: ',nlocs + close(21) + +!c read in the files + open(21,file=ufile,access='direct',recl=len*8*lines) + read(21,rec=1)unwrapped + close(21) + open(21,file=demfile,access='direct',recl=len*2*lines) + read(21,rec=1)dem + close(21) + if(iargc().ge.7)then + open(21,file=intfile,access='direct',recl=len*8*lines) + read(21,rec=1)igram + close(21) + end if + +!c set up the regression + do i=1,nlocs + phase(i)=unwrapped(loc(1,i)+len,loc(2,i)) + height(i)=dem(loc(1,i),loc(2,i)) +! if(i.lt.10)print *,i,loc(:,i),height(i),phase(i) + end do + +!c regression coeffs + call linfit(height,phase,nlocs,poly) +! print *,'regression polynomial ',poly + +!c predict phase from height +! do i=1,nlocs +! print *,'regress predicts ',i,loc(:,i),phase(i),height(i),height(i)*poly(2)+poly(1) +! end do + +!c and apply it to the file + do i=1,lines + do j=1,len + unwrapped(j+len,i)=unwrapped(j+len,i)-dem(j,i)*poly(2)-poly(1) + if(iargc().ge.7)then + phasecorr=-dem(j,i)*poly(2)-poly(1) + igram(j,i)=igram(j,i)*cmplx(cos(phasecorr),sin(phasecorr)) + end if + end do + end do + +!c save regressed files + open(21,file=outfile,access='direct',recl=len*8*lines) + write(21,rec=1)unwrapped + close(21) + if(iargc().ge.7)then + open(21,file=trim(intfile)//'.reg',access='direct',recl=len*8*lines) + write(21,rec=1)igram + close(21) + end if + end + + + subroutine linfit(xin,yin,ndata,poly) + +! Polynomial to be fitted: +! Y = a(0) + a(1).X + a(2).X^2 + ... + a(m).X^m + + USE lsq + IMPLICIT NONE + + REAL*8 :: x(10000000),y(10000000),xrow(0:20), wt = 1.0, beta(0:20), & + var, covmat(231), sterr(0:20), totalSS, center + real*4 poly(3) + REAL*4 :: xin(ndata), yin(ndata) + INTEGER :: i, ier, iostatus, j, m, n, ndata + LOGICAL :: fit_const = .TRUE., lindep(0:20), xfirst + + do i=1,ndata + y(i)=yin(i) + x(i)=xin(i) + end do + + n=ndata + m=1 + +! Least-squares calculations + + CALL startup(m, fit_const) + DO i = 1, n + xrow(0) = 1.0 + DO j = 1, m + xrow(j) = x(i) * xrow(j-1) + END DO + CALL includ(wt, xrow, y(i)) + END DO + + CALL sing(lindep, ier) + IF (ier /= 0) THEN + DO i = 0, m + IF (lindep(i)) WRITE(*, '(a, i3)') ' Singularity detected for power: ', i + END DO + END IF + +! Calculate progressive residual sums of squares + CALL ss() + var = rss(m+1) / (n - m - 1) + +! Calculate least-squares regn. coeffs. + CALL regcf(beta, m+1, ier) + +! Calculate covariance matrix, and hence std. errors of coeffs. + CALL cov(m+1, var, covmat, 231, sterr, ier) + poly(1)=beta(0) + poly(2)=beta(1) + poly(3)=beta(2) + +!!$ WRITE(*, *) 'Least-squares coefficients & std. errors' +!!$ WRITE(*, *) 'Power Coefficient Std.error Resid.sum of sq.' +!!$ DO i = 0, m +!!$ WRITE(*, '(i4, g20.12, " ", g14.6, " ", g14.6)') & +!!$ i, beta(i), sterr(i), rss(i+1) +!!$ END DO +!!$ +!!$ WRITE(*, *) +!!$ WRITE(*, '(a, g20.12)') ' Residual standard deviation = ', SQRT(var) +!!$ totalSS = rss(1) +!!$ WRITE(*, '(a, g20.12)') ' R^2 = ', (totalSS - rss(m+1))/totalSS + return + + end subroutine linfit diff --git a/kp_scripts/util/unwrap_igrams_mask.py b/kp_scripts/util/unwrap_igrams_mask.py new file mode 100644 index 0000000..393f7a8 --- /dev/null +++ b/kp_scripts/util/unwrap_igrams_mask.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 + +# unwrap_igrams - unwrap set of interferograms + +import sys +ver=sys.version_info + +import os +import sys +import math + +if len(sys.argv) < 4: + print ('Usage: unwrap_igrams_mask.py intlist len path_to_aux ') + sys.exit(1) + +### --- REQUIRED PARAMETERS --- ### +filelist=sys.argv[1] +width=sys.argv[2] +path_to_aux=sys.argv[3] + +### --- DEFAULT PARAMETERS --- ### +intmaskflag = 'N' +scenemask = 'none' +box=1 + +### --- CHECK PARAMETERS --- ### +args = len(sys.argv) +if args>4: + intmaksflag = sys.argv[4] + +if args>5: + scenemask = sys.argv[5] + +if args>6: + box=sys.argv[6] + +print(filelist,width,box) + +with open(filelist,'r') as fint: + intlist = fint.readlines() + +for line in intlist: + intfile=line.strip() + print ('intfile: ',intfile) + + shortname = intfile.split('/')[-1] + + # Get other file names + unwfile=intfile.replace('.int','.unw') + lowpassfile=intfile+'.lowpass' + cfile=path_to_aux+'/'+shortname.replace('.int','.cc') + + # Determine masking + if intmaskflag == 'Y': + mfile = ' -M '+path_to_aux+'/'+shortname.replace('.int','.mask') + elif scenemask != 'none': + mfile = ' -M '+scenemask + elif scenemask == 'none': + mfile = '' + + if abs(float(box)-1) > 0.1: + # lowpass filter a bit to aid unwrapping is desired + ret=os.system('$PROC_HOME/ps/lowpass '+intfile+' '+width+' '+box) + ret=os.system('$PROC_HOME/bin/snaphu '+lowpassfile+' '+width+' -d -o '+unwfile+' -c '+cfile+' '+mfile+' --mcf') + else: + ret=os.system('$PROC_HOME/bin/snaphu '+intfile+' '+width+' -d -o '+unwfile+' -c '+cfile+' '+mfile+' --mcf') + +sys.exit() + diff --git a/kp_scripts/util/unwrap_parallel.py b/kp_scripts/util/unwrap_parallel.py new file mode 100644 index 0000000..9a753b9 --- /dev/null +++ b/kp_scripts/util/unwrap_parallel.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +# +# +# unwrap_parallel.py -- unwrap multiple interferograms in parallel, up to 20 at a time +# +# note: assume that unwlist, intlist, geolist all exist +# + +import os +import subprocess +import sys +import glob + +if len(sys.argv) <2: + print ('Usage: unwrap_parallel.py intlist width ') + sys.exit(0) + +### --- REQUIRED PARAMETERS --- ### +intfiles = sys.argv[1] +unwwidth = sys.argv[2] + +### --- DEFAULT PARAMETERS --- ### +path_to_aux = '.' +intmaskflag = 'N' +scenemask = 'none' +box = '1' + +### --- CHECK INPUTS --- ### +args = len(sys.argv) +if args>3: + path_to_aux = sys.argv[3] + +if args>4: + intmaskflag = sys.argv[4] + +if args>5: + scenemask = sys.argv[5] + +if args>6: + box = sys.argv[6] + +### --- RUN SCRIPT --- ### + +# Read in the list of intlists +with open(intfiles,'r') as fint: + ints = fint.readlines() + +# compare intlist to unwlist and write out new intlist with files that have yet to be unwrapped +int_unw = [] +with open('intlist_for_unwrapping','w') as fint: + unws = glob.glob('*.unw*') + for int_file in ints: + name = int_file.strip().replace('.int','.unw') + # name = name.replace('.unwerp','') + if name not in unws: + fint.write(int_file) + int_unw.append(int_file.strip()) +intfiles = 'intlist_for_unwrapping' + +if len(int_unw)>0: + # unwrap the interferograms, first creating multiple lists of them + print('\n Splitting input intlist into multiple files') + command = '$PROC_HOME/util/splitintlist.py '+intfiles # split igrams into many lists + print(' '+command) + ret = os.system(command) + + list_int = [] + for k in range(30): + list_int.append('intlist'+str(k)) + + # unwrap in parallel + num=0 + unwcommand=[] + prochome = os.getenv('PROC_HOME') + for intlist in list_int: + if os.path.getsize(intlist) > 0: + command = prochome+'/kp_scripts/util/unwrap_igrams_mask.py '+intlist+' '+unwwidth+' '+path_to_aux+' '+intmaskflag+' '+scenemask+' '+box + print(' '+command) + unwcommand.append(subprocess.Popen([prochome+'/kp_scripts/util/unwrap_igrams_mask.py',intlist,unwwidth,path_to_aux,intmaskflag,scenemask,box])) + num=num+1 + + for i in range(num): + unwcommand[i].wait() + + print('\n Interferograms unwrapped!') + + print(' Cleaning Directory') + for k in range(30): + ret=os.system('rm intlist'+str(k)) + +else: + print('\n All interferograms already unwrapped!') + +command = 'rm intlist_for_unwrapping' +print(' '+command) +ret=os.system(command) + + diff --git a/ps/Makefile b/ps/Makefile new file mode 100644 index 0000000..25e5cc7 --- /dev/null +++ b/ps/Makefile @@ -0,0 +1,14 @@ +CC=g++ -std=c++11 -Wall +#PYTHON_LIB=`python3-config --includes` +#PYTHON_EXT=`python3-config --extension-suffix` + +default: All + +sario.o: sario.hpp sario.cpp + $(CC) -c sario.cpp + +All: sario.o psinterp.cpp + $(CC) -fopenmp -O3 -fPIC -c psinterp.cpp + $(CC) -fopenmp -O3 -fPIC psinterp.o sario.o -o psinterp +clean: + rm -f *.o psinterp diff --git a/ps/all_similarity_mask b/ps/all_similarity_mask new file mode 100644 index 0000000..0d0a926 Binary files /dev/null and b/ps/all_similarity_mask differ diff --git a/ps/cosine_sim b/ps/cosine_sim new file mode 100644 index 0000000..09b6462 Binary files /dev/null and b/ps/cosine_sim differ diff --git a/ps/cosine_sim.f90.old b/ps/cosine_sim.f90.old new file mode 100644 index 0000000..0459174 --- /dev/null +++ b/ps/cosine_sim.f90.old @@ -0,0 +1,669 @@ +! run ps detection using cosine simiuarlity on a set of wrapped files +! Read also correlation and amp files +! this version allows for averaging multiple reference points if file is supplied + +PROGRAM cosine_sim + use omp_lib + + IMPLICIT none + + !specifications + INTEGER::i,j,r,c,rows,cols,stat,fstat,ierr,looks,k,kk,n_refs,igramnumber,iter + real*8 jdprimary, jdsecondary + INTEGER*8 ::nr, naz, reclphase, recsize !image size + INTEGER::nslc !number of slcs + INTEGER::ncells !number of cells (files in flist) + INTEGER,DIMENSION(13)::statb + CHARACTER(200)::filelist,str,ref_locs_file, outfile + CHARACTER(200),DIMENSION(:),ALLOCATABLE::cells !array of file names + CHARACTER(200)::strint,strunw,stramp,strcc + REAL,DIMENSION(:,:,:),ALLOCATABLE::phase !,coh,temp3 + REAL,DIMENSION(:,:),ALLOCATABLE::amp,dat,mask,temp2,stack,stacktime,scr,similarity,allsimilarity + real, dimension(:,:,:),allocatable :: amps + complex*8,dimension(:,:,:),allocatable :: igrams + real*4 psthresh,simthresh,similaritymean + integer*1,dimension(:,:),allocatable :: bytemask + + !executions + + !flist: list of wrapped intrerferogram files + !nfiles: number of files in flist + !nslcs: number of slc files + !len: width of the files + IF(iargc().lt.3)THEN + WRITE(*,*)'usage: cosine_sim igramlist len outfile ' + STOP + END IF + + CALL getarg(1,filelist) + CALL getarg(2,str) + READ(str,*)nr !width of files + call getarg(3,outfile) + psthresh=2. + simthresh=0.5 + if(iargc().ge.4)then + call getarg(4,str) + read(str,*)psthresh + end if + if(iargc().ge.5)then + call getarg(5,str) + read(str,*)simthresh + end if + + ALLOCATE(cells(10000)) + + OPEN(UNIT=1,FILE=filelist,STATUS='old') + do ncells=1,10000 + READ(1,'(A)',end=10,IOSTAT=stat)cells(ncells) + end do +! PRINT*,cells(1:10) !print file names +10 continue + CLOSE(1) + ncells=ncells-1 + print *,'Number of interferograms: ',ncells + + OPEN(UNIT=2,FILE=cells(1),FORM='unformatted',ACCESS='direct',RECL=nr*8) + ierr=fstat(2,statb) + naz=statb(8)/8/nr + WRITE(*,*)'Lines in file: ',naz + CLOSE(2) + + ALLOCATE(igrams(nr,naz,ncells),scr(nr,naz),similarity(nr,naz),allsimilarity(nr,naz)) + ALLOCATE(amp(nr,naz),amps(nr,naz,ncells),bytemask(nr,naz))!,npts(nr,naz),stacktime(nr,naz),stack(nr,naz)) + + igrams(:,:,:)=0 + amp(:,:)=0 + + !read in wrapped igrams + !$omp parallel do private(strint) shared(igrams,ncells,amps) + DO i=1,ncells + strint=cells(i) + OPEN(UNIT=i+10,FILE=strint,FORM='unformatted',ACCESS='direct',RECL=2*nr*naz*4,CONVERT='LITTLE_ENDIAN') + READ(i+10,rec=1)igrams(:,:,i) + CLOSE(i+10) + amps(:,:,i)=cabs(igrams(:,:,i))**2 + END DO + !$omp end parallel do + amp=sqrt(sum(amps,3)/ncells) ! save amplitude average for later use if desired + + ! compute scr for each point and free up some memory + deallocate (amps) + + r=nr + c=naz + call mlesub(igrams,r,c,ncells,scr) + allocate(phase(nr,naz,ncells)) + phase=atan2(aimag(igrams),real(igrams)) + deallocate (igrams) + +!Save nr, naz, nslc, ncells parameters in one file + OPEN(15,FILE='cosine_sim_parameters',STATUS='replace') + WRITE(15,*) nr, naz, nslc, ncells + CLOSE(15) + + PRINT*,'Mle pass complete' + ! save scr for later analysis + open(15,file='scr_floats',access='stream') + write(15)scr + close(15) + open(15,file='scr',access='stream') + do i=1,naz + write(15)amp(:,i),scr(:,i) + end do + close(15) + print *,'Saved scr estimates as floats (scr_floats) and mht (scr) formats' + + ! save byte mask for scr's above threshold + bytemask=0 + k=0 + do j=1,naz + do i=1,nr + if(scr(i,j).ge.psthresh)then + bytemask(i,j)=-1 + k=k+1 + end if + end do + end do + open(15,file='scr_mask',access='stream') + write(15)bytemask + close(15) + print *,'Saved scr mask as scr_mask, number= ',k + + ! compute median_similarity - for each ps pixel (>th), compute its phase similarity with + ! nearby PS and return the median of similarity measurements + + call median_similarity(phase,scr,r,c,ncells,psthresh,similarity) + ! save median similarity for later analysis + open(15,file='median_similarity',access='stream') + write(15)similarity + close(15) + print *,'Saved median similarity (median_similarity) as floats with ps threshold ',psthresh + + ! save byte mask for median similarity above threshold + bytemask=0 + k=0 + similaritymean=0. + do j=1,naz + do i=1,nr + if(similarity(i,j).ge.simthresh)then + bytemask(i,j)=-1 + k=k+1 + similaritymean=similaritymean+similarity(i,j) + end if + end do + end do + similaritymean=similaritymean/k + open(15,file='median_sim_mask',access='stream') + write(15)bytemask + close(15) + print *,'Saved median similarity mask as median_sim_mask, number, mean= ',k,similaritymean + + ! and now get the similarity everywhere, adding in newly discovered points + call all_similarity(phase,similarity,r,c,ncells,simthresh,allsimilarity) + + do iter=1,3 + print *,'Iteration ',iter + similarity=allsimilarity + call all_similarity(phase,similarity,r,c,ncells,simthresh,allsimilarity) + print *,'finished ',iter + end do + + ! save in mht format + open(15,file=outfile,access='stream') + do i=1,naz + write(15)amp(:,i),allsimilarity(:,i) + end do + ! and also as floats + open(15,file='all_similarity',access='stream') + write(15)allsimilarity + close(15) + print *,'Saved final similarity as floats (allsimilarity) and mht ',trim(outfile),' with sim threshold ',simthresh + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +!FUNCTIONS + +CONTAINS + + !strrep(str,orig,rep): replaces all instaces of orig in str with rep + CHARACTER(30) FUNCTION strrep(str,orig,rep) + IMPLICIT none + !specifications + INTEGER::i + CHARACTER(100),INTENT(IN)::str + CHARACTER(2),INTENT(IN)::orig,rep + !executions + strrep(:) = str(:) + DO i=1,LEN(strrep)-1 + IF(strrep(i:i+1) == orig(:))THEN + strrep(i:i+1) = rep(:) + END IF + END DO + END FUNCTION strrep + +! ------------------ +FUNCTION Replace_Text (s,text,rep) RESULT(outs) +CHARACTER(*) :: s,text,rep +CHARACTER(LEN(s)+100) :: outs ! provide outs with extra 100 char len +INTEGER :: i, nt, nr + +outs = s ; nt = LEN_TRIM(text) ; nr = LEN_TRIM(rep) +DO + i = INDEX(outs,text(:nt)) ; IF (i == 0) EXIT + outs = outs(:i-1) // rep(:nr) // outs(i+nt:) +END DO +END FUNCTION Replace_Text + + !mean2d(mat): calculates the mean value of a 2D matrix mat with r rows and c columns + REAL FUNCTION mean2d(mat,rows,cols) + IMPLICIT none + REAL,INTENT(IN),DIMENSION(:,:) :: mat + INTEGER :: r,c,rows,cols + mean2d=0 + DO r=1,rows + DO c=1,cols + mean2d=mean2d+mat(r,c) + END DO + END DO + mean2d=mean2d/SIZE(mat) + END FUNCTION mean2d + + subroutine mlesub(igrams,len,lines,nigrams,scrout) +!c mlestack - apply mle estimator to phases from stacked time series of igrams + + use omp_lib + implicit none + complex*8, allocatable :: in(:,:),igram(:,:),igramfilt(:,:),filt(:,:) + complex*8 csum,igrams(len,lines,nigrams) + complex*8, allocatable :: valid(:) + real*4, allocatable :: out(:),phase(:) + real*4 scrout(len,lines) + real*4 pdfs(200,100),prob(100) + real*4 pi,rho,rhoest,thresh,probmax,pow,ph,screst,x,y,dx,aveps,avemag,box,aveph,amp,scr + integer*4, allocatable :: iph(:) + character*100 flist,fin(10000),fout,str + integer statb(13),fstat,len,lines,nigrams,ibox,nvalid,kmax,krho,ixfft,iyfft,iy,ix,line,ixbins + + !$omp parallel + ! n=omp_get_num_threads() + !$omp end parallel + !print *, 'Max threads used: ', n + + ! default to box size 10 + box=10. + + print *,'Interferograms analyzed: ',nigrams + +!c get sizes for ffts + do i=1,20 + if(len.gt.2**i)ixfft=2**i + if(lines.gt.2**i)iyfft=2**i + end do + ixfft=ixfft*2 + iyfft=iyfft*2 + print *,'FFT sizes: ',ixfft,iyfft + +!c allocate memory + allocate(out(len*2)) + allocate(in(len,nigrams)) + allocate(phase(nigrams)) + allocate(iph(nigrams)) + allocate(valid(nigrams)) + allocate(filt(ixfft,iyfft)) + +!c precompute slc pdfs as function of rho and theta + ixbins=200 + pi=4*atan2(1.,1.) + dx=2*pi/ixbins + + do k=1,100 + rho=(k-1)/100. + scr=1/(1./rho-1.) +! print *,rho,scr + do i=1,ixbins + ph=-pi+dx/2.+(i-1)*dx + pdfs(i,k)=1./2./pi*exp(-scr*(sin(ph)**2))*(exp(-scr*cos(ph)**2)+ & + sqrt(pi*scr)*cos(ph)*(1-erf(-sqrt(scr)*cos(ph)))) + end do + end do + +!c make a filter directly in frequency domain + do ix=1,ixfft + if(ix.le.ixfft/2)then + x=sinc((ix-1)/(ixfft/box)) + else + x=sinc((ixfft-ix+1)/(ixfft/box)) + end if + + do iy=1,iyfft + if(iy.le.iyfft/2)then + y=sinc((iy-1)/(iyfft/box)) + else + y=sinc((iyfft-iy+1)/(iyfft/box)) + end if + + filt(ix,iy)=cmplx(x*y, 0.) + + end do + end do + + filt=conjg(filt) + +!c save filter for debug purposes +! print *,'Writing filter, sizes: ',ixfft,iyfft +! open(24,file='filter',access='direct',recl=ixfft*iyfft*8) +! write(24,rec=1)filt +! close(24) +! open(99,file='inmlesub',access='stream') +! write(99)igrams(:,:,1) +! close(99) + print *,'Filtering interferograms ' + !$omp parallel do private(i,igram,igramfilt,kk,k) & + !$omp shared(nigrams,lines,len,filt,ixfft,iyfft,fin,igrams) + do i=1,nigrams + if(mod(i,10).eq.1)print *,'Filtering igram ',i + allocate(igramfilt(ixfft,iyfft)) + igramfilt=cmplx(0.,0.) + igramfilt(1:len,1:lines)=igrams(:,:,i) + call fft2d(igramfilt,ixfft,iyfft,-1) + igramfilt=igramfilt*filt + call fft2d(igramfilt,ixfft,iyfft,1) + igramfilt(1:len,1:lines)=igramfilt(1:len,1:lines)/cabs(igramfilt(1:len,1:lines)) + do k=1,len + do kk=1,lines + if(isnan(real(igramfilt(k,kk))).or.isnan(aimag(igramfilt(k,kk))))then + igramfilt(k,kk)=cmplx(0.,0.) + end if + end do + end do + igrams(1:len,1:lines,i)=igrams(1:len,1:lines,i)*conjg(igramfilt(1:len,1:lines))!/cabs(igramfilt(1:len,1:lines)) + deallocate (igramfilt) + end do + !$omp end parallel do + +!c open output files for debug if desired +! fout='q' +! open(20,file=fout,form='unformatted',access='direct',recl=len*8) +! open(21,file=trim(fout)//'psonly',form='unformatted',access='direct',recl=len*4) + +!c loop over lines + + !$omp parallel do private(line,in,csum,i,k,pow,amp,phase,aveph) & + !$omp private(iph,probmax,krho,prob,kmax,rhoest,screst,out) & + !$omp private(nvalid,valid,thresh,avemag) & + !$omp shared(lines,nigrams,len,pi,ixbins,pdfs,igrams,scrout) + do line=1,lines + if(mod(line,100).eq.0)print *,'At line: ',line + + !c read line from each interferogram + do i=1,nigrams + in(:,i)=igrams(:,line,i) +! read(100+i,rec=line)in(:,i) + end do + + !c loop over pixels in the line + do k=1,len + !c make a list of valid points + avemag=0. + do i=1,nigrams + avemag=avemag+cabs(in(k,i)) + end do + thresh=avemag/1000. + nvalid=0 + do i=1,nigrams + if (cabs(in(k,i)).ge.thresh)then + nvalid=nvalid+1 + valid(nvalid)=in(k,i) + end if + end do + !c force each sequence to zero mean + csum=sum(valid(1:nvalid)) + csum=csum/cabs(csum) + valid(1:nvalid)=valid(1:nvalid)*conjg(csum) + + !c sqrt(average powers) to store as amplitude value + pow=0. + pow=sum(cabs(valid(1:nvalid))**2) + amp=sqrt(pow/nvalid) + if(isnan(amp))amp=0. + !c get the phase series + phase(1:nvalid)=atan2(aimag(valid(1:nvalid)),real(valid(1:nvalid))) +! print *,valid(1:nvalid) + !c apply mle + aveph=sum(phase(1:nvalid)) + aveph=aveph/nvalid + do i=1,nvalid + iph(i)=(phase(i)+pi)/2./pi*ixbins+1 + if(iph(i).lt.-100000)iph(i)=1 + end do + probmax=-1.e12 + !!!$omp parallel do private(krho,prob,i) + !!!$omp+ shared(pdfs,iph,probmax,nvalid) + do krho=1,100 + prob(krho)=0. + do i=1,nvalid + prob(krho)=prob(krho)+alog(pdfs(iph(i),krho)+1.e-12) + end do + if(prob(krho).ge.probmax)then + kmax=krho + probmax=prob(krho) + end if + end do + !!!$omp end parallel do + rhoest=(kmax-1)/100. + screst=1./(1./rhoest-1.) + + !c save result for pixel + out(k)=amp + out(k+len)=screst + + end do + + !c write out line +! write(20,rec=line)out +! write(21,rec=line)(out(k),k=len+1,len*2) + scrout(:,line)=out(len+1:len*2) + end do + !$omp end parallel do + end subroutine mlesub + + real function sinc(q) + real*4 q,pi + parameter (pi=3.14159265359) + + if(abs(q).le.1.e-6)then + sinc=1. + else + sinc=sin(pi*q)/pi/q + end if + return + end function sinc + + subroutine fft2d(arr,ixfft,iyfft,dir) + + integer*4 ixfft,iyfft,dir + complex*8 arr(ixfft,iyfft) + integer*8 planf, plani + real*4, allocatable :: work(:) + + allocate(work(max(ixfft,iyfft)*4+15)) + + if(dir.eq.-1)then + !print *,'forward start',ixfft,iyfft + call fftw2d_f77_create_plan(planf, ixfft, iyfft, -1, 8) +! call fftwnd_f77_one(planf, arr, planfwork) + call fftwnd_f77_one(planf, arr, work) + call fftw_f77_destroy_plan(planf) + !print *,'forward end' + end if + + if(dir.eq.1)then + !print *,'reverse start' + call fftw2d_f77_create_plan(plani, ixfft, iyfft, +1, 8) +! call fftwnd_f77_one(plani, arr, planiwork) + call fftwnd_f77_one(plani, arr, work) + call fftw_f77_destroy_plan(plani) + !print *,'reverse end' + end if + + deallocate (work) + + return + end subroutine fft2d + + subroutine median_similarity(phase,scr,len,lines,nigrams,psthresh,similarity) + !c median similarity of each ps point with neighbors in some distance range + ! compute median_similarity - for each ps pixel (>th), compute its phase similarity with + ! nearby PS and return the median of similarity measurements + + use omp_lib + implicit none + + real*4 scr(len,lines),similarity(len,lines),phase(len,lines,nigrams) + real*4 rmin, rmax, psthresh, dist, cossim(20) + integer len,lines,i,j,ii,jj,nigrams,irmin,irmax,neigh,k,boxsize + integer*4, allocatable :: iindex(:),jindex(:) + real*4, allocatable :: r(:) + + irmin=2 + irmax=50 + boxsize=2*irmax+1 + + ! set up spiral scanning + allocate (iindex(boxsize*boxsize),jindex(boxsize*boxsize),r(boxsize*boxsize)) + + call spiralscan(boxsize,iindex,jindex,r) + + ! loop over all pixels and find ps + !$omp parallel do private(i,neigh,cossim,dist,ii,jj) & + !$omp shared(lines,len,boxsize,iindex,jindex,r,irmin,irmax,similarity,psthresh,nigrams,allsimilarity) + + do j=1,lines + if(mod(j,100).eq.1)print *,'Computing similarity at line ',j + do i=1,len + allsimilarity(i,j)=0. + if(similarity(i,j).ge.simthresh)then + allsimilarity(i,j)=similarity(i,j) + else + ! find neighbor ps + neigh=0 + cossim=0. + do k=1,boxsize*boxsize + if(neigh.lt.20)then !search until we get 20 neighbors + ii=i+iindex(k) + jj=j+jindex(k) + dist=r(k) + ! do ii=-irmax,irmax + ! do jj=-irmax,irmax + ! dist=(ii*ii+jj*jj) + ! if(dist.ge.irmin**2.and.dist.le.irmax**2)then ! in ring to check + if(dist.ge.irmin.and.dist.le.irmax)then + if(ii.ge.1.and.ii.le.len.and.jj.ge.1.and.jj.le.lines)then ! in bounds + if(similarity(ii,jj).ge.simthresh)then ! found a neighbor + neigh=neigh+1 + if(neigh.le.20)then + cossim(neigh)=sum(cos(phase(i,j,:)-phase(ii,jj,:)))/nigrams + ! else + ! exit + end if + end if ! end threshold test + end if ! end in-bounds test + end if ! end distance test + end if ! end test if reached 20 neighbors + end do + ! if(neigh.ge.20)exit + ! end do + ! allsimilarity(i,j)=0. + if(neigh.ge.1)then + if(neigh.eq.1)then + allsimilarity(i,j)=cossim(1) + else + allsimilarity(i,j)=sum(cossim(1:neigh))/neigh + end if + ! print *,neigh,i,j,similarity(i,j) + end if + end if ! finished looking for neighbors option + end do ! end of pixel loop + end do !end of line loop + !$omp end parallel do +! open(21,file='all_sim',access='stream') ! for dbug if desired +! write(21)allsimilarity +! close(21) + + ! count number above threshold after computation + k=0 + do j=1,lines + do i=1,len + if(allsimilarity(i,j).ge.simthresh)k=k+1 + end do + end do + print *,'Number of similarity pixels found: ',k + + return + end subroutine all_similarity + + SUBROUTINE SORT(N,RA) + integer N,IR,J,L,I + real*4 ra(N),rra + +! DIMENSION RA(N) + L=N/2+1 + IR=N +10 CONTINUE + IF(L.GT.1)THEN + L=L-1 + RRA=RA(L) + ELSE + RRA=RA(IR) + RA(IR)=RA(1) + IR=IR-1 + IF(IR.EQ.1)THEN + RA(1)=RRA + RETURN + ENDIF + ENDIF + I=L + J=L+L +20 IF(J.LE.IR)THEN + IF(J.LT.IR)THEN + IF(RA(J).LT.RA(J+1))J=J+1 + ENDIF + IF(RRA.LT.RA(J))THEN + RA(I)=RA(J) + I=J + J=J+J + ELSE + J=IR+1 + ENDIF + GO TO 20 + ENDIF + RA(I)=RRA + GO TO 10 + END + + subroutine spiralscan(boxsize,iindex,jindex,r) + + integer boxsize,x,y,n,count + integer*4 iindex(boxsize*boxsize),jindex(boxsize*boxsize) + integer*4, allocatable :: array(:,:),scan(:,:),xx(:) + real*4 r(boxsize*boxsize) + + allocate (array(boxsize,boxsize),scan(boxsize,boxsize),xx(boxsize)) + do i=1,boxsize + xx(i)=i-boxsize/2-1 + end do + + x=0 + y=1 + n=0 + count=boxsize; + do i = 1,count + x = x + 1 + array(x,y) = n + n = n + 1 + end do + do + count = count - 1 + do i = 1,count + y = y + 1 + array(x,y) = n + n = n + 1 + end do + do i = 1,count + x = x - 1 + array(x,y) = n + n = n + 1 + end do + if (n > boxsize*boxsize-1) exit + count = count - 1 + do i = 1,count + y = y - 1 + array(x,y) = n + n = n + 1 + end do + do i = 1,count + x = x + 1 + array(x,y) = n + n = n + 1 + end do + if (n > boxsize*boxsize-1) exit + end do + scan=boxsize*boxsize-array + +!c create list of indices in scan order + do i=1,boxsize + do j=1,boxsize + iindex(scan(i,j))=xx(i) + jindex(scan(i,j))=xx(j) + end do + end do + !c the distance between center point and each scanned pixels + do k = 1, boxsize*boxsize + r(k) = sqrt(real(iindex(k))**2.+real(jindex(k))**2.) + end do + + return + end subroutine spiralscan + +END PROGRAM cosine_sim + diff --git a/ps/cosine_sim.f90~ b/ps/cosine_sim.f90~ new file mode 100644 index 0000000..cba395c --- /dev/null +++ b/ps/cosine_sim.f90~ @@ -0,0 +1,793 @@ +! run ps detection using cosine similarlity on a set of wrapped files +! Read also correlation and amp files +! this version allows for averaging multiple reference points if file is supplied +! modified to use fftw3 28jan23 + +PROGRAM cosine_sim + use omp_lib + + IMPLICIT none + + !specifications + INTEGER::i,j,r,c,rows,cols,stat,fstat,ierr,looks,k,kk,n_refs,igramnumber,iter + real*8 jdprimary, jdsecondary + INTEGER*8 ::nr, naz, reclphase, recsize, planf, plani !image size + INTEGER::ncells !number of cells (files in flist) + INTEGER,DIMENSION(13)::statb + CHARACTER(200)::filelist,str,ref_locs_file, outfile + CHARACTER(200),DIMENSION(:),ALLOCATABLE::cells !array of file names + CHARACTER(200)::strint,strunw,stramp,strcc + REAL,DIMENSION(:,:,:),ALLOCATABLE::phase,amps !,coh,temp3 + REAL,DIMENSION(:,:),ALLOCATABLE::amp,dat,mask,temp2,stack,stacktime,scr,similarity,allsimilarity + complex*8,dimension(:,:,:),allocatable :: igrams + real*4 psthresh,simthresh,similaritymean + integer*1,dimension(:,:),allocatable :: bytemask + + !executions + + !flist: list of wrapped intrerferogram files + !nfiles: number of files in flist + !nslcs: number of slc files + !len: width of the files + IF(iargc().lt.3)THEN + WRITE(*,*)'usage: cosine_sim igramlist len outfile ' + STOP + END IF + + CALL getarg(1,filelist) + CALL getarg(2,str) + READ(str,*)nr !width of files + call getarg(3,outfile) + psthresh=2. + simthresh=0.5 + if(iargc().ge.4)then + call getarg(4,str) + read(str,*)psthresh + end if + if(iargc().ge.5)then + call getarg(5,str) + read(str,*)simthresh + end if + + ALLOCATE(cells(10000)) + + OPEN(UNIT=1,FILE=filelist,STATUS='old') + do ncells=1,10000 + READ(1,'(A)',end=10,IOSTAT=stat)cells(ncells) + end do +! PRINT*,cells(1:10) !print file names +10 continue + CLOSE(1) + ncells=ncells-1 + + OPEN(UNIT=2,FILE=cells(1),FORM='unformatted',ACCESS='direct',RECL=nr*8) + ierr=fstat(2,statb) + naz=statb(8)/8/nr + CLOSE(2) + print *,'Number of interferograms: ',ncells,', Lines per file: ',naz + + ALLOCATE(igrams(nr,naz,ncells),scr(nr,naz),similarity(nr,naz),allsimilarity(nr,naz)) + ALLOCATE(amp(nr,naz),bytemask(nr,naz),amps(nr,naz,ncells)) + + !read in wrapped igrams + !$omp parallel do private(strint) shared(igrams,ncells,amps) + DO i=1,ncells + strint=cells(i) + OPEN(UNIT=i+10,FILE=strint,FORM='unformatted',ACCESS='direct',RECL=2*nr*naz*4,CONVERT='LITTLE_ENDIAN') + READ(i+10,rec=1)igrams(:,:,i) + CLOSE(i+10) + amps(:,:,i)=cabs(igrams(:,:,i))**2 +! amp=amp+cabs(igrams(:,:,i))**2 + END DO + !$omp end parallel do + amp=sqrt(sum(amps,3)/ncells) ! save amplitude average for later use if desred + deallocate (amps) + + ! first step is to get candidate points from mle + ! compute scr for each point and free up some memory + r=nr + c=naz + call mlesub(igrams,r,c,ncells,scr) + allocate(phase(nr,naz,ncells)) + !$omp parallel do shared(phase,igrams,ncells) + do i=1,ncells + phase(:,:,i)=atan2(aimag(igrams(:,:,i)),real(igrams(:,:,i))) + end do + !$omp end parallel do + deallocate (igrams) + +!Save nr, naz, nslc, ncells parameters in one file + OPEN(15,FILE='cosine_sim_parameters',STATUS='replace') + WRITE(15,*) nr, naz, ncells, psthresh, simthresh + CLOSE(15) + + PRINT*,'Mle pass complete' + ! save scr for later analysis + open(15,file='scr_floats',access='stream') + write(15)scr + close(15) + open(15,file='scr',access='stream') + do i=1,naz + write(15)amp(:,i),scr(:,i) + end do + close(15) + print *,'Saved scr estimates as floats (scr_floats) and mht (scr) formats' + + ! save byte mask for scr's above threshold + bytemask=0 + k=0 + do j=1,naz + do i=1,nr + if(scr(i,j).ge.psthresh)then + bytemask(i,j)=-1 + k=k+1 + end if + end do + end do + open(15,file='scr_mask',access='stream') + write(15)bytemask + close(15) + print *,'Saved scr mask as scr_mask, number= ',k + + ! step 2. remove false positives by cosine similarity filter + ! compute median_similarity - for each ps pixel (>th), compute its phase similarity with + ! nearby PS and return the median of similarity measurements + + call median_similarity(phase,scr,r,c,ncells,psthresh,similarity) + ! save median similarity for later analysis + open(15,file='median_similarity',access='stream') + write(15)similarity + close(15) + print *,'Saved median similarity (median_similarity) as floats with ps threshold ',psthresh + + ! save byte mask for median similarity above threshold + bytemask=0 + k=0 + similaritymean=0. + do j=1,naz + do i=1,nr + if(similarity(i,j).ge.simthresh)then + bytemask(i,j)=-1 + k=k+1 + similaritymean=similaritymean+similarity(i,j) + end if + end do + end do + similaritymean=similaritymean/k + open(15,file='median_sim_mask',access='stream') + write(15)bytemask + close(15) + print *,'Saved median similarity mask as median_sim_mask, number, mean= ',k,similaritymean + + ! step 3. scan all ppoints for similarity with refined ps set, adding extra discoveries + ! and now get the similarity everywhere, adding in newly discovered points +! call all_similarity(phase,similarity,r,c,ncells,simthresh,allsimilarity) + + do iter=1,3 + print *,'Iteration ',iter + if(iter.gt.1)similarity=allsimilarity + call all_similarity(phase,similarity,r,c,ncells,simthresh,allsimilarity) +! print *,'finished ',iter + end do + + ! save in mht format + open(15,file=outfile,access='stream') + do i=1,naz + write(15)amp(:,i),allsimilarity(:,i) + end do + ! and also as floats + open(15,file='all_similarity',access='stream') + write(15)allsimilarity + close(15) + print *,'Saved final similarity as floats (all_similarity) and mht ',trim(outfile),' with sim threshold ',simthresh + ! and finally a byte mask over the sim_thresh + bytemask=0 + k=0 + similaritymean=0. + kk=0 + do j=1,naz + do i=1,nr + if(allsimilarity(i,j).ge.simthresh)then + bytemask(i,j)=-1 + k=k+1 + similaritymean=similaritymean+similarity(i,j) + end if + if(abs(allsimilarity(i,j)).ge.1.e-3)then + kk=kk+1 + end if + end do + end do + similaritymean=similaritymean/k + open(15,file='all_similarity_mask',access='stream') + write(15)bytemask + close(15) + print *,'Saved all_similarity mask as all_similarity_mask, number, mean= ',k,similaritymean + print *,'Fractional coverage: ',float(kk)/float(naz)/float(nr) + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +!FUNCTIONS + +CONTAINS + + !strrep(str,orig,rep): replaces all instaces of orig in str with rep + CHARACTER(30) FUNCTION strrep(str,orig,rep) + IMPLICIT none + !specifications + INTEGER::i + CHARACTER(100),INTENT(IN)::str + CHARACTER(2),INTENT(IN)::orig,rep + !executions + strrep(:) = str(:) + DO i=1,LEN(strrep)-1 + IF(strrep(i:i+1) == orig(:))THEN + strrep(i:i+1) = rep(:) + END IF + END DO + END FUNCTION strrep + +! ------------------ +FUNCTION Replace_Text (s,text,rep) RESULT(outs) +CHARACTER(*) :: s,text,rep +CHARACTER(LEN(s)+100) :: outs ! provide outs with extra 100 char len +INTEGER :: i, nt, nr + +outs = s ; nt = LEN_TRIM(text) ; nr = LEN_TRIM(rep) +DO + i = INDEX(outs,text(:nt)) ; IF (i == 0) EXIT + outs = outs(:i-1) // rep(:nr) // outs(i+nt:) +END DO +END FUNCTION Replace_Text + + !mean2d(mat): calculates the mean value of a 2D matrix mat with r rows and c columns + REAL FUNCTION mean2d(mat,rows,cols) + IMPLICIT none + REAL,INTENT(IN),DIMENSION(:,:) :: mat + INTEGER :: r,c,rows,cols + mean2d=0 + DO r=1,rows + DO c=1,cols + mean2d=mean2d+mat(r,c) + END DO + END DO + mean2d=mean2d/SIZE(mat) + END FUNCTION mean2d + + subroutine mlesub(igrams,len,lines,nigrams,scrout) +!c mlestack - apply mle estimator to phases from stacked time series of igrams + + use omp_lib + implicit none + complex*8, allocatable :: in(:,:),igram(:,:),igramfilt(:,:),filt(:,:),work(:,:) + complex*8 csum,igrams(len,lines,nigrams) + complex*8, allocatable :: valid(:) + real*4, allocatable :: out(:),phase(:) + real*4 scrout(len,lines) + real*4 pdfs(200,100),prob(100) + real*4 pi,rho,rhoest,thresh,probmax,pow,ph,screst,x,y,dx,aveps,avemag,box,aveph,amp,scr + integer*4, allocatable :: iph(:) + character*100 flist,fin(10000),fout,str + integer statb(13),fstat,len,lines,nigrams,ibox,nvalid,kmax,krho,ixfft,iyfft,iy,ix,line,ixbins + + !$omp parallel + ! n=omp_get_num_threads() + !$omp end parallel + !print *, 'Max threads used: ', n + + ! default to box size 10 + box=10. + +! print *,'Interferograms analyzed: ',nigrams + +!c get sizes for ffts + do i=1,20 + if(len.gt.2**i)ixfft=2**i + if(lines.gt.2**i)iyfft=2**i + end do + ixfft=ixfft*2 + iyfft=iyfft*2 +! print *,'FFT sizes: ',ixfft,iyfft + +!c allocate memory + allocate(out(len*2)) + allocate(in(len,nigrams)) + allocate(phase(nigrams)) + allocate(iph(nigrams)) + allocate(valid(nigrams)) + allocate(filt(ixfft,iyfft)) + +!c precompute slc pdfs as function of rho and theta + ixbins=200 + pi=4*atan2(1.,1.) + dx=2*pi/ixbins + + do k=1,100 + rho=(k-1)/100. + scr=1/(1./rho-1.) +! print *,rho,scr + do i=1,ixbins + ph=-pi+dx/2.+(i-1)*dx + pdfs(i,k)=1./2./pi*exp(-scr*(sin(ph)**2))*(exp(-scr*cos(ph)**2)+ & + sqrt(pi*scr)*cos(ph)*(1-erf(-sqrt(scr)*cos(ph)))) + end do + end do + +!c make a filter directly in frequency domain + do ix=1,ixfft + if(ix.le.ixfft/2)then + x=sinc((ix-1)/(ixfft/box)) + else + x=sinc((ixfft-ix+1)/(ixfft/box)) + end if + + do iy=1,iyfft + if(iy.le.iyfft/2)then + y=sinc((iy-1)/(iyfft/box)) + else + y=sinc((iyfft-iy+1)/(iyfft/box)) + end if + + filt(ix,iy)=cmplx(x*y, 0.) + + end do + end do + + filt=conjg(filt) + +!c save filter for debug purposes +! print *,'Writing filter, sizes: ',ixfft,iyfft +! open(24,file='filter',access='direct',recl=ixfft*iyfft*8) +! write(24,rec=1)filt +! close(24) +! open(99,file='inmlesub',access='stream') +! write(99)igrams(:,:,1) +! close(99) + print *,'Filtering interferograms ' + allocate(igramfilt(ixfft,iyfft)) + call sfftw_plan_dft_2d(planf,ixfft,iyfft,igramfilt,igramfilt,-1,64) + call sfftw_plan_dft_2d(plani,ixfft,iyfft,igramfilt,igramfilt,+1,64) + deallocate(igramfilt) + print *,'plans created' + !$omp parallel do private(i,igram,igramfilt,kk,k) & + !$omp shared(nigrams,lines,len,filt,ixfft,iyfft,fin,igrams,planf,plani) + do i=1,nigrams + if(mod(i,100).eq.1)print *,'Filtering igram ',i + allocate(igramfilt(ixfft,iyfft)) + igramfilt=cmplx(0.,0.) + igramfilt(1:len,1:lines)=igrams(:,:,i) + print *,'calling first fft2d',i + call sfftw_execute_dft(planf,igramfilt,igramfilt) + +! call fft2d(igramfilt,ixfft,iyfft,-1) + print *,'forward ',i + igramfilt=igramfilt*filt + + call sfftw_execute_dft(plani,igramfilt,igramfilt) +! call fft2d(igramfilt,ixfft,iyfft,1) + print *,'inverse ',i + igramfilt(1:len,1:lines)=igramfilt(1:len,1:lines)/cabs(igramfilt(1:len,1:lines)) + do k=1,len + do kk=1,lines + if(isnan(real(igramfilt(k,kk))).or.isnan(aimag(igramfilt(k,kk))))then + igramfilt(k,kk)=cmplx(0.,0.) + end if + end do + end do + igrams(1:len,1:lines,i)=igrams(1:len,1:lines,i)*conjg(igramfilt(1:len,1:lines))!/cabs(igramfilt(1:len,1:lines)) + deallocate (igramfilt) + end do + !$omp end parallel do + call sfftw_destroy_plan(planf) + call sfftw_destroy_plan(plani) + +!c open output files for debug if desired +! fout='q' +! open(20,file=fout,form='unformatted',access='direct',recl=len*8) +! open(21,file=trim(fout)//'psonly',form='unformatted',access='direct',recl=len*4) + +!c loop over lines + !$omp parallel do private(line,in,csum,i,k,pow,amp,phase,aveph) & + !$omp private(iph,probmax,krho,prob,kmax,rhoest,screst,out) & + !$omp private(nvalid,valid,thresh,avemag) & + !$omp shared(lines,nigrams,len,pi,ixbins,pdfs,igrams,scrout) + do line=1,lines + if(mod(line,1000).eq.1)print *,'At line: ',line + + !c read line from each interferogram + do i=1,nigrams + in(:,i)=igrams(:,line,i) +! read(100+i,rec=line)in(:,i) + end do + + !c loop over pixels in the line + do k=1,len + !c make a list of valid points + avemag=0. + do i=1,nigrams + avemag=avemag+cabs(in(k,i)) + end do + thresh=avemag/1000. + nvalid=0 + do i=1,nigrams + if (cabs(in(k,i)).ge.thresh)then + nvalid=nvalid+1 + valid(nvalid)=in(k,i) + end if + end do + !c force each sequence to zero mean + csum=sum(valid(1:nvalid)) + csum=csum/cabs(csum) + valid(1:nvalid)=valid(1:nvalid)*conjg(csum) + + !c sqrt(average powers) to store as amplitude value + pow=0. + pow=sum(cabs(valid(1:nvalid))**2) + amp=sqrt(pow/nvalid) + if(isnan(amp))amp=0. + !c get the phase series + phase(1:nvalid)=atan2(aimag(valid(1:nvalid)),real(valid(1:nvalid))) +! print *,valid(1:nvalid) + !c apply mle + aveph=sum(phase(1:nvalid)) + aveph=aveph/nvalid + do i=1,nvalid + iph(i)=(phase(i)+pi)/2./pi*ixbins+1 + if(iph(i).lt.-100000)iph(i)=1 + end do + probmax=-1.e12 + !!!$omp parallel do private(krho,prob,i) + !!!$omp+ shared(pdfs,iph,probmax,nvalid) + do krho=1,100 + prob(krho)=0. + do i=1,nvalid + prob(krho)=prob(krho)+alog(pdfs(iph(i),krho)+1.e-12) + end do + if(prob(krho).ge.probmax)then + kmax=krho + probmax=prob(krho) + end if + end do + !!!$omp end parallel do + rhoest=(kmax-1)/100. + screst=1./(1./rhoest-1.) + + !c save result for pixel + out(k)=amp + out(k+len)=screst + + end do + + !c write out line +! write(20,rec=line)out +! write(21,rec=line)(out(k),k=len+1,len*2) + scrout(:,line)=out(len+1:len*2) + end do + !$omp end parallel do + return + end subroutine mlesub + + real function sinc(q) + real*4 q,pi + parameter (pi=3.14159265359) + + if(abs(q).le.1.e-6)then + sinc=1. + else + sinc=sin(pi*q)/pi/q + end if + return + end function sinc + + subroutine fft2d(arr,ixfft,iyfft,dir) + + integer*4 ixfft,iyfft,dir + complex*8 arr(ixfft,iyfft) + integer*8 planf, plani + + if(dir.eq.dir)return + if(dir.eq.-1)then +! print *,'forward start',ixfft,iyfft +! call fftw2d_f77_create_plan(planf, ixfft, iyfft, -1, 8) +! call fftwnd_f77_one(planf, arr, work) +! call fftw_f77_destroy_plan(planf) + call sfftw_plan_dft_2d(planf,ixfft,iyfft,arr,arr,-1,64) + call sfftw_execute_dft(planf,arr,arr) + call sfftw_destroy_plan(planf) +! print *,'forward end' + end if + + if(dir.eq.1)then +! print *,'reverse start' +! call fftw2d_f77_create_plan(plani, ixfft, iyfft, +1, 8) +! call fftwnd_f77_one(plani, arr, work) +! call fftw_f77_destroy_plan(plani) + call sfftw_plan_dft_2d(plani,ixfft,iyfft,arr,arr,+1,64) + call sfftw_execute_dft(plani,arr,arr) + call sfftw_destroy_plan(plani) +! print *,'reverse end' + end if + + return + end subroutine fft2d + + subroutine median_similarity(phase,scr,len,lines,nigrams,psthresh,similarity) + !c median similarity of each ps point with neighbors in some distance range + ! compute median_similarity - for each ps pixel (>th), compute its phase similarity with + ! nearby PS and return the median of similarity measurements + + use omp_lib + implicit none + + real*4 scr(len,lines),similarity(len,lines),phase(len,lines,nigrams) + real*4 rmin, rmax, psthresh, dist, cossim(20) + integer len,lines,i,j,ii,jj,nigrams,irmin,irmax,neigh,k,boxsize + integer*4, allocatable :: iindex(:),jindex(:) + real*4, allocatable :: r(:) + + irmin=2 + irmax=50 + boxsize=2*irmax+1 + + ! set up spiral scanning + allocate (iindex(boxsize*boxsize),jindex(boxsize*boxsize),r(boxsize*boxsize)) + + call spiralscan(boxsize,iindex,jindex,r) + + ! how many are we starting with? + k=0 + do j=1,lines + do i=1,len + if(scr(i,j).ge.psthresh)k=k+1 + end do + end do +! print *,'Starting median pass with number of ps = ',k + + ! loop over all ps pixels and compute median similarity + !$omp parallel do private(j,k,neigh,cossim,dist,ii,jj) & + !$omp shared(lines,len,boxsize,iindex,jindex,r,irmin,irmax,scr,psthresh,nigrams,similarity,phase) + do i=1,len + do j=1,lines + similarity(i,j)=0. + if(scr(i,j).ge.psthresh)then ! found an scr ps candidate + ! find neighbor ps + neigh=0 + cossim=0. + do k=1,boxsize*boxsize + if(neigh.lt.20)then ! only continue if less than 20 neighbors found + ii=i+iindex(k) + jj=j+jindex(k) + dist=r(k) + ! do ii=-irmax,irmax + ! do jj=-irmax,irmax + ! dist=(ii*ii+jj*jj) + if(dist.ge.irmin.and.dist.le.irmax)then ! in ring to check + if(ii.ge.1.and.ii.le.len.and.jj.ge.1.and.jj.le.lines)then ! in bounds + + if(scr(ii,jj).ge.psthresh)then ! found a neighbor ps candidate + neigh=neigh+1 + if(neigh.le.20)then + cossim(neigh)=sum(cos(phase(i,j,:)-phase(ii,jj,:)))/nigrams + !else + ! exit + end if + end if ! end threshold test + end if ! end in-bounds test + end if ! end distance test + end if ! end less than 20 test + ! if(neigh.ge.20)exit + end do + ! similarity(i,j)=0. + if(neigh.ge.1)then ! if we found neighbor ps candidates + if(neigh.eq.1)then + similarity(i,j)=cossim(1) + else + call sort(neigh,cossim) + similarity(i,j)=cossim(neigh/2) + end if +! print *,i,j,neigh,similarity(i,j),cossim + end if + end if ! end initial candidate test + end do ! end line loop + end do ! end pixel loop +! open(21,file='median_sim',access='stream') ! for debug if desired +! write(21)similarity +! close(21) + + return + end subroutine median_similarity + + subroutine all_similarity(phase,similarity,len,lines,nigrams,simthresh,allsimilarity) + ! all_similarity - return similarity for all pixels vis a vis ps points + ! only add newly discovered pixels, do not remove previous ones + + use omp_lib + implicit none + + real*4 allsimilarity(len,lines),similarity(len,lines),phase(len,lines,nigrams) + real*4 rmin, rmax, dist, cossim(20),simthresh + integer len,lines,i,j,ii,jj,nigrams,irmin,irmax,neigh,boxsize,k + integer*4, allocatable :: iindex(:),jindex(:) + real*4, allocatable :: r(:) + + irmin=2 + irmax=50 + boxsize=2*irmax+1 + + ! set up spiral scanning + allocate (iindex(boxsize*boxsize),jindex(boxsize*boxsize),r(boxsize*boxsize)) + + call spiralscan(boxsize,iindex,jindex,r) + + ! loop over all pixels and find ps + !$omp parallel do private(i,neigh,cossim,dist,ii,jj) & + !$omp shared(lines,len,boxsize,iindex,jindex,r,irmin,irmax,similarity,psthresh,nigrams,allsimilarity) + + do j=1,lines + if(mod(j,1000).eq.1)print *,'Computing similarity at line ',j + do i=1,len + allsimilarity(i,j)=0. + if(similarity(i,j).ge.simthresh)then + allsimilarity(i,j)=similarity(i,j) + else + ! find neighbor ps + neigh=0 + cossim=0. + do k=1,boxsize*boxsize + if(neigh.lt.20)then !search until we get 20 neighbors + ii=i+iindex(k) + jj=j+jindex(k) + dist=r(k) + ! do ii=-irmax,irmax + ! do jj=-irmax,irmax + ! dist=(ii*ii+jj*jj) + ! if(dist.ge.irmin**2.and.dist.le.irmax**2)then ! in ring to check + if(dist.ge.irmin.and.dist.le.irmax)then + if(ii.ge.1.and.ii.le.len.and.jj.ge.1.and.jj.le.lines)then ! in bounds + if(similarity(ii,jj).ge.simthresh)then ! found a neighbor + neigh=neigh+1 + if(neigh.le.20)then + cossim(neigh)=sum(cos(phase(i,j,:)-phase(ii,jj,:)))/nigrams + ! else + ! exit + end if + end if ! end threshold test + end if ! end in-bounds test + end if ! end distance test + end if ! end test if reached 20 neighbors + end do + ! if(neigh.ge.20)exit + ! end do + ! allsimilarity(i,j)=0. + if(neigh.ge.1)then + if(neigh.eq.1)then + allsimilarity(i,j)=cossim(1) + else + allsimilarity(i,j)=sum(cossim(1:neigh))/neigh + end if + ! print *,neigh,i,j,similarity(i,j) + end if + end if ! finished looking for neighbors option + end do ! end of pixel loop + end do !end of line loop + !$omp end parallel do +! open(21,file='all_sim',access='stream') ! for dbug if desired +! write(21)allsimilarity +! close(21) + + ! count number above threshold after computation + k=0 + do j=1,lines + do i=1,len + if(allsimilarity(i,j).ge.simthresh)k=k+1 + end do + end do + print *,'Number of similarity pixels found: ',k + + return + end subroutine all_similarity + + SUBROUTINE SORT(N,RA) + integer N,IR,J,L,I + real*4 ra(N),rra + +! DIMENSION RA(N) + L=N/2+1 + IR=N +10 CONTINUE + IF(L.GT.1)THEN + L=L-1 + RRA=RA(L) + ELSE + RRA=RA(IR) + RA(IR)=RA(1) + IR=IR-1 + IF(IR.EQ.1)THEN + RA(1)=RRA + RETURN + ENDIF + ENDIF + I=L + J=L+L +20 IF(J.LE.IR)THEN + IF(J.LT.IR)THEN + IF(RA(J).LT.RA(J+1))J=J+1 + ENDIF + IF(RRA.LT.RA(J))THEN + RA(I)=RA(J) + I=J + J=J+J + ELSE + J=IR+1 + ENDIF + GO TO 20 + ENDIF + RA(I)=RRA + GO TO 10 + END + + subroutine spiralscan(boxsize,iindex,jindex,r) + + integer boxsize,x,y,n,count + integer*4 iindex(boxsize*boxsize),jindex(boxsize*boxsize) + integer*4, allocatable :: array(:,:),scan(:,:),xx(:) + real*4 r(boxsize*boxsize) + + allocate (array(boxsize,boxsize),scan(boxsize,boxsize),xx(boxsize)) + do i=1,boxsize + xx(i)=i-boxsize/2-1 + end do + + x=0 + y=1 + n=0 + count=boxsize; + do i = 1,count + x = x + 1 + array(x,y) = n + n = n + 1 + end do + do + count = count - 1 + do i = 1,count + y = y + 1 + array(x,y) = n + n = n + 1 + end do + do i = 1,count + x = x - 1 + array(x,y) = n + n = n + 1 + end do + if (n > boxsize*boxsize-1) exit + count = count - 1 + do i = 1,count + y = y - 1 + array(x,y) = n + n = n + 1 + end do + do i = 1,count + x = x + 1 + array(x,y) = n + n = n + 1 + end do + if (n > boxsize*boxsize-1) exit + end do + scan=boxsize*boxsize-array + +!c create list of indices in scan order + do i=1,boxsize + do j=1,boxsize + iindex(scan(i,j))=xx(i) + jindex(scan(i,j))=xx(j) + end do + end do + !c the distance between center point and each scanned pixels + do k = 1, boxsize*boxsize + r(k) = sqrt(real(iindex(k))**2.+real(jindex(k))**2.) + end do + + return + end subroutine spiralscan + +END PROGRAM cosine_sim + diff --git a/ps/make_simmasks.py b/ps/make_simmasks.py new file mode 100644 index 0000000..552a49a --- /dev/null +++ b/ps/make_simmasks.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 + +# make_simmasks.py -- create ps mask for an input intlist, and many masks given optional input geolist + +import sys +from datetime import datetime +import os +import math + +if len(sys.argv) < 3: + print ('Usage: make_simmasks.py igramlist length ') + sys.exit(1) + +igramlist=sys.argv[1] +length=sys.argv[2] + +geolist = 'none' +if len(sys.argv)>3: + geolist = sys.argv[3] + +psthresh='2' +simthresh='0.4' +if len(sys.argv) > 4: + psthresh=sys.argv[4] +if len(sys.argv) > 5: + simthresh=sys.argv[5] + +if geolist != 'none': + fgeos=open(geolist,'r') + geos=fgeos.readlines() + fgeos.close() + + for line in geos: + words=line.split('/') + indate=words[-1][:8] + intlist = 'intlist_'+indate + print('\n##### ----- '+intlist+' ----- #####') + # find the ps and store in various formats + command="$PROC_HOME/ps/cosine_sim "+intlist+' '+length+' pssim '+psthresh+' '+simthresh + print(command) + ret=os.system(command) + command = 'mv all_similarity_mask all_similarity_mask_'+indate + ret=os.system(command) + +# find the ps and store in various formats +command="$PROC_HOME/ps/cosine_sim "+igramlist+" "+length+" pssim "+psthresh+" "+simthresh +print(command) +ret=os.system(command) + diff --git a/ps/psfilter.py b/ps/psfilter.py new file mode 100644 index 0000000..cff9f3d --- /dev/null +++ b/ps/psfilter.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 + +# psfilter.py -- create ps mask and interpolate ps from a second list + +import sys +from datetime import datetime +import os +import math + +if len(sys.argv) < 3: + print ('Usage: psfilter.py igramlist interpolatelist length ') + sys.exit(1) + +igramlist=sys.argv[1] +interpolatelist=sys.argv[2] +length=sys.argv[3] +psthresh='2' +simthresh='0.4' +if len(sys.argv) > 4: + psthresh=sys.argv[4] +if len(sys.argv) > 5: + simthresh=sys.argv[5] + +# find the ps and store in various formats +command="$PROC_HOME/ps/cosine_sim igramlist "+length+" pssim "+psthresh+" "+simthresh +print(command) +ret=os.system(command) + +# interpolate the files in the interpolatelist +finterp=open(interpolatelist,'r') +interp=finterp.readlines() +finterp.close() + +for intfile in interp: + command="$PROC_HOME/ps/psinterp "+intfile.strip()+" all_similarity_mask "+intfile.strip()+".interp "+length + print(command) + ret=os.system(command) diff --git a/ps/psfilter.py~ b/ps/psfilter.py~ new file mode 100644 index 0000000..aceb6b1 --- /dev/null +++ b/ps/psfilter.py~ @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 + +# psfilter.py -- create ps mask and interpolate ps from a second list + +import sys +from datetime import datetime +import os +import math + +if len(sys.argv) < 3: + print ('Usage: psfilter.py igramlist interpolatelist length ') + sys.exit(1) + +igramlist=sys.argv[1] +interpolatelist=sys.argv[2] +length=sys.argv[3] +psthresh='2' +simthresh='0.4' +if len(sys.argv) > 3: + psthresh=sys.argv[4] +if len(sys.argv) > 4: + simthresh=sys.argv[5] + +# find the ps and store in various formats +command="$PROC_HOME/ps/cosine_sim igramlist "+length+" pssim "+psthresh+" "+simthresh +print(command) +ret=os.system(command) + +# interpolate the files in the interpolatelist +finterp=open(interpolatelist,'r') +interp=finterp.readlines() +finterp.close() + +for intfile in interp: + command="$PROC_HOME/ps/psinterp "+intfile.strip()+" all_similarity_mask "+intfile.strip()+".interp "+length + print(command) + ret=os.system(command) diff --git a/ps/psinterp b/ps/psinterp new file mode 100644 index 0000000..1ad5ccc Binary files /dev/null and b/ps/psinterp differ diff --git a/ps/psinterp.cpp b/ps/psinterp.cpp new file mode 100644 index 0000000..5e8c68d --- /dev/null +++ b/ps/psinterp.cpp @@ -0,0 +1,164 @@ +#include +#include +#include +#include +#include +#include "sario.hpp" + +std::vector> scan_array( + const unsigned int rdmin, + const unsigned int rdmax){ + // using the mid-point cirlce drawing algorithm to search for neighboring PS pixels + // code adpated from "https://www.geeksforgeeks.org/mid-point-circle-drawing-algorithm/" + int x,y,p,flag; //flag > 0 means there are holes between concentric circles + std::vector> visited( + rdmax, + std::vector (rdmax,false)); + std::vector> indices; + visited[0][0] = true; + for (int r = 1; r < (int)rdmax; ++r){ + x = r; + y = 0; + p = 1-r; + if (r > (int)rdmin){ + indices.push_back(std::vector {r,0}); + indices.push_back(std::vector {-r,0}); + indices.push_back(std::vector {0,r}); + indices.push_back(std::vector {0,-r}); + } + visited[r][0] = true; + visited[0][r] = true; + flag = 0; + while (x>y){ + // do not need to fill holes + if (flag == 0){ + y++; + if (p <= 0){ + // Mid-point is inside or on the perimeter + p += 2*y + 1; + }else{ + // Mid-point is outside the perimeter + x--; + p += 2*y - 2*x + 1; + } + }else{ + flag--; + } + + // All the perimeter points have already been visited + if (x (int)rdmin){ + indices.push_back(std::vector {x,y}); + indices.push_back(std::vector {-x,-y}); + indices.push_back(std::vector {x,-y}); + indices.push_back(std::vector {-x,y}); + if (x != y){ + indices.push_back(std::vector {y,x}); + indices.push_back(std::vector {-y,-x}); + indices.push_back(std::vector {y,-x}); + indices.push_back(std::vector {-y,x}); + } + } + if (flag > 0){ + x++; + } + } + } + return indices; +} + +std::vector>> interp_( + const std::vector>> &ifg, + const std::vector> &ps, + const unsigned int N, + const unsigned int rdmax, + const double alpha){ + unsigned int nrow, ncol, nindices; + nrow = ps.size(); + ncol = ps[0].size(); + std::vector>> ifg_interp( + nrow,std::vector>(ncol,0.)); + std::vector> indices; + indices = scan_array(0,rdmax); + nindices = indices.size(); + /* + for (size_t i=0; i csum = 0; + std::vector r2(N,0.); + std::vector> cphase(N,0.); + for (std::size_t i = 0; i < nindices; ++i){ + r = r0 + indices[i][0]; + c = c0 + indices[i][1]; + if ((r >= 0) && (r < (int)nrow) && (c >= 0) && (c < (int)ncol) && ps[r][c]){ + // calculate the square distance to the center pixel + r2[counter] = std::pow(indices[i][0],2)+std::pow(indices[i][1],2); + cphase[counter] = ifg[r][c]/(1e-12+std::abs(ifg[r][c])); + ++counter; + if (counter >= N){ + break; + } + } + } + for (std::size_t i = 0; i < counter; ++i){ + csum += std::exp(-r2[i]/2/std::pow(r2[counter-1],alpha))*cphase[i]; + } + ifg_interp[r0][c0] = std::abs(ifg[r0][c0])*csum/(std::abs(csum)+1e-12); + } + } + return ifg_interp; +} + +void interp( + const std::string &ifgfile, + const std::string &psfile, + const std::string &outputfile, + const unsigned int nrow, + const unsigned int ncol, + const unsigned int N, + const unsigned int rdmax, + const double alpha){ + std::vector>> ifg = read_ifg( + ifgfile,nrow,ncol); + std::vector> ps = read_psfile( + psfile,nrow,ncol); + std::vector>> ifg_interp = interp_( + ifg,ps,N,rdmax,alpha); + save_ifg(ifg_interp, outputfile); + return; +} + +int main(int argc, char *argv[]){ + const std::string ifgfile = "20170213_20180115.int"; + const std::string psfile = "all_similarity_mask"; + const std::string outputfile = "20170213_20180115.interp.int"; + interp(ifgfile,psfile,outputfile,600,600,20,101,0.75); + return 0; +} + +#define STRINGIFY(x) #x +#define MACRO_STRINGIFY(x) STRINGIFY(x) + diff --git a/ps/psinterp.f90~ b/ps/psinterp.f90~ new file mode 100644 index 0000000..7cc50a6 --- /dev/null +++ b/ps/psinterp.f90~ @@ -0,0 +1,279 @@ +!***** +! +! psinterp algorithm from Wang and Chen (2022) translated to f90 +! + use omp_lib + + implicit none + character*300 intfile, psfile, outfile, str + integer nps, ibox, len, rdmin, rdmax, i, j, neigh, lines, k, jj, ii, irmin, irmax + real alpha + complex*8, dimension(:,:), allocatable :: igram, out + integer*1, dimension(:,:), allocatable :: psmask + integer boxsize,nvalid + integer, allocatable :: iindex(:),jindex(:) + real, allocatable :: r(:),r2(:) + integer statb(13),fstat,ierr + complex*8, allocatable :: cphase(:) + complex*8 csum + + if(iargc().lt.4)then + print *,'Usage: psinterp intfile psfile outfile len' + call exit + end if + + ! some hardwired params to be refined later + nps=20 ! number of ps to use for spiral + ibox=101 ! max box size + alpha=1. ! exponent parameter for distance weighting + rdmin=3 ! min scan distance + rdmax=50 ! max scan distance + + call getarg(1,intfile) + call getarg(2,psfile) + call getarg(3,outfile) + call getarg(4,str) + read(str,*)len + + ! compute the scan indices array + irmax=50 + boxsize=2*irmax+1 + allocate (iindex(boxsize*boxsize),jindex(boxsize*boxsize),r(boxsize*boxsize)) + nvalid=boxsize*boxsize +! call spiralscan2(boxsize,iindex,jindex,r)!,nvalid) +! do i=1,20 +! print *,i,iindex(i),jindex(i) +! end do +! print *,'Spiral indices set' + call spiralscan(boxsize,iindex,jindex,r,nvalid) +! do i=1,nvalid +! print *,i,iindex(i),jindex(i) +! end do + print *,'Spiral indices set',nvalid +! if(i.eq.i)call exit +! call spiralscan2(boxsize,iindex,jindex,r) +!print *,iindex +!print *,jindex + + ! read in int and ps files, open output file + open(21,file=intfile,access='stream') + ierr=fstat(21,statb) + lines=statb(8)/len/8 + allocate (igram(len,lines),psmask(len,lines),out(len,lines)) + out=cmplx(0.,0.) + + read(21)igram + close(21) + open(21,file=psfile,access='stream') + read(21)psmask + close(21) + print *,'data read in' + + ! now start the interpolation + + allocate (r2(nps),cphase(nps)) + !$omp parallel do private(i,j,csum,r2,cphase,neigh,k,ii,jj) & + !$omp shared(lines,len,iindex,jindex,r,igram,alpha,out) + do j=1,lines + do i=1,len + csum=cmplx(0.,0.) + r2=0. + cphase=cmplx(0.,0.) + neigh=0 + do k=1,nvalid + ii=i+iindex(k) + jj=j+jindex(k) + if(ii.ge.1.and.ii.le.len.and.jj.ge.1.and.jj.le.lines)then ! in bounds + if(psmask(ii,jj).ne.0)then + neigh=neigh+1 + if(neigh.gt.nps)go to 11 + r2(neigh)=r(k)*r(k) + cphase(neigh)=igram(ii,jj)/(1.e-12+abs(igram(ii,jj))) + end if + end if ! end bounds check + end do ! end scan of box +11 neigh=neigh-1 + ! sum the phases with weights +! print *,cphase + if(neigh.gt.0)then + do k=1,neigh + csum=csum+exp(-r2(k)/2/r2(neigh)**alpha)*cphase(k) + end do + out(i,j)=abs(igram(i,j))*csum/(1.e-12+abs(csum)) + !print *,'out ',out(i,j),igram(i,j) +! print *,r2(1:neigh) +! print *,r2(1:neigh)/r2(neigh) +! print *,i,j,exp(-r2(1:neigh)/2/r2(neigh)) + end if + end do + end do + !$omp end parallel do + + open(31,file=outfile,access='stream') + write(31)out + close(31) + + +end program + + subroutine spiralscan(boxsize,iindex,jindex,r,nvalid) + + integer boxsize,x,y,n,count,irmin,irmax,p,k,nvalid + integer*4 iindex(boxsize*boxsize),jindex(boxsize*boxsize) + real*4 r(boxsize*boxsize) + integer*1, allocatable :: visited(:,:) + + allocate (visited(boxsize,boxsize)) + + irmin=0 + irmax=boxsize/2 + visited(1,1)=-1 + k=1 + + do i=1,irmax + x=i + y=0 + p=1-i + if(i.gt.irmin)then + iindex(k)=i + jindex(k)=0 + k=k+1 + iindex(k)=-i + jindex(k)=0 + k=k+1 + iindex(k)=0 + jindex(k)=i + k=k+1 + iindex(k)=0 + jindex(k)=-i + k=k+1 + end if + visited(i+1,1)=-1 + visited(1,i+1)=-1 + flag=0 + + do while(x.gt.y) + if(flag.eq.0)then + y=y+1 + if(p.le.0)then + p=p+2*y+1 + else + x=x-1 + p=p+2*y-2*x+1 + end if + else + flag=flag-1 + end if + if(x.lt.y)go to 12 + do while(visited(x-1+1,y+1).eq.0) + x=x-1 + flag=flag+1 + end do + visited(x+1,y+1)=-1 + visited(y+1,x+1)=-1 + if(i.gt.irmin)then + iindex(k)=x + jindex(k)=y + k=k+1 + iindex(k)=-x + jindex(k)=-y + k=k+1 + iindex(k)=x + jindex(k)=-y + k=k+1 + iindex(k)=-x + jindex(k)=y + k=k+1 + if(x.ne.y)then + iindex(k)=y + jindex(k)=x + k=k+1 + iindex(k)=-y + jindex(k)=-x + k=k+1 + iindex(k)=y + jindex(k)=-x + k=k+1 + iindex(k)=-y + jindex(k)=x + k=k+1 + end if + end if + if(flag>0)x=x+1 + end do +12 continue + end do + nvalid=k-1 + do k = 1, nvalid + r(k) = sqrt(real(iindex(k))**2.+real(jindex(k))**2.) + end do + + return + end subroutine spiralscan + + +! alternate spiral algorithm using squares intead of circles + subroutine spiralscan2(boxsize,iindex,jindex,r) + + integer boxsize,x,y,n,count + integer*4 iindex(boxsize*boxsize),jindex(boxsize*boxsize) + integer*4, allocatable :: array(:,:),scan(:,:),xx(:) + real*4 r(boxsize*boxsize) + + allocate (array(boxsize,boxsize),scan(boxsize,boxsize),xx(boxsize)) + do i=1,boxsize + xx(i)=i-boxsize/2-1 + end do + + x=0 + y=1 + n=0 + count=boxsize; + do i = 1,count + x = x + 1 + array(x,y) = n + n = n + 1 + end do + do + count = count - 1 + do i = 1,count + y = y + 1 + array(x,y) = n + n = n + 1 + end do + do i = 1,count + x = x - 1 + array(x,y) = n + n = n + 1 + end do + if (n > boxsize*boxsize-1) exit + count = count - 1 + do i = 1,count + y = y - 1 + array(x,y) = n + n = n + 1 + end do + do i = 1,count + x = x + 1 + array(x,y) = n + n = n + 1 + end do + if (n > boxsize*boxsize-1) exit + end do + scan=boxsize*boxsize-array + +!c create list of indices in scan order + do i=1,boxsize + do j=1,boxsize + iindex(scan(i,j))=xx(i) + jindex(scan(i,j))=xx(j) + end do + end do + !c the distance between center point and each scanned pixels + do k = 1, boxsize*boxsize + r(k) = sqrt(real(iindex(k))**2.+real(jindex(k))**2.) + end do + + return + end subroutine spiralscan2 + diff --git a/ps/psinterp.o b/ps/psinterp.o new file mode 100644 index 0000000..6178186 Binary files /dev/null and b/ps/psinterp.o differ diff --git a/ps/pssim b/ps/pssim new file mode 100644 index 0000000..76234c6 Binary files /dev/null and b/ps/pssim differ diff --git a/ps/qlist b/ps/qlist new file mode 100644 index 0000000..a977a19 --- /dev/null +++ b/ps/qlist @@ -0,0 +1 @@ +20170213_20180115.int diff --git a/ps/qout b/ps/qout new file mode 100644 index 0000000..b449c04 Binary files /dev/null and b/ps/qout differ diff --git a/ps/sario.cpp b/ps/sario.cpp new file mode 100644 index 0000000..7d8aafa --- /dev/null +++ b/ps/sario.cpp @@ -0,0 +1,70 @@ +#include +#include +#include +#include +#include + +#include "sario.hpp" + +std::vector>> read_ifg( + const std::string& fname, + const std::size_t nrow, + const std::size_t ncol){ + using comp_double = std::complex; + std::size_t re_idx, im_idx; + std::vector imgbuffer(nrow*ncol*2); + std::ifstream fin(fname, std::ios::binary); + fin.read((char *)imgbuffer.data(), nrow*ncol*2*sizeof(float)); + fin.close(); + std::vector> ifg(nrow,std::vector(ncol,0)); + for (std::size_t r = 0; r < nrow; ++r){ + for (std::size_t c = 0; c < ncol; ++c){ + re_idx = ncol*2*r + c*2; + im_idx = ncol*2*r + c*2+1; + ifg[r][c] = + std::complex(imgbuffer[re_idx],imgbuffer[im_idx]); + } + } + return ifg; +} + +void save_ifg( + const std::vector>> ifg, + const std::string& fname){ + std::ofstream fout(fname, std::ios::out | std::ios::binary); + if (!fout.is_open()){ + printf("Unable to open file %s\n",fname.c_str()); + return; + } + for (std::size_t i = 0; i < ifg.size(); ++i){ + std::vector line(2*ifg[i].size(),0.); + for (std::size_t j=0; j < ifg[i].size(); ++j){ + line[2*j] = float(std::real(ifg[i][j])); + line[2*j+1] = float(std::imag(ifg[i][j])); + } + fout.write((char*)&line[0],line.size()*sizeof(float)); + } + fout.close(); + return; +} + +std::vector> read_psfile( + const std::string& fname, + const std::size_t nrow, + const std::size_t ncol){ + std::vector imgbuffer(nrow*ncol); + std::ifstream fin(fname, std::ios::in| std::ios::out| std::ios::binary); + fin.read((char *)imgbuffer.data(), nrow*ncol); + fin.close(); + std::vector> ps(nrow,std::vector(ncol,false)); + int counter = 0; + for (std::size_t i = 0; i < nrow; ++i){ + for (std::size_t j = 0; j < ncol; ++j){ + ps[i][j] = bool(imgbuffer[ncol*i + j]); + if(ps[i][j]){ + counter++; + } + } + } + return ps; +} diff --git a/ps/sario.hpp b/ps/sario.hpp new file mode 100644 index 0000000..85e1b3a --- /dev/null +++ b/ps/sario.hpp @@ -0,0 +1,23 @@ +#ifndef SARIO +#define SARIO + +#include +#include +#include + +std::vector>> read_ifg( + const std::string& fname, + const std::size_t nrow, + const std::size_t ncol); + +void save_ifg( + const std::vector>> ifg, + const std::string& fname); + +std::vector> read_psfile( + const std::string& fname, + const std::size_t nrow, + const std::size_t ncol); + +#endif + diff --git a/ps/sario.o b/ps/sario.o new file mode 100644 index 0000000..4bdeca2 Binary files /dev/null and b/ps/sario.o differ diff --git a/ps/spiralscan b/ps/spiralscan new file mode 100644 index 0000000..780dc3b Binary files /dev/null and b/ps/spiralscan differ diff --git a/ps/spiralscan.f90 b/ps/spiralscan.f90 new file mode 100644 index 0000000..6946b88 --- /dev/null +++ b/ps/spiralscan.f90 @@ -0,0 +1,83 @@ + integer boxsize + integer, allocatable :: iindex(:),jindex(:) + real, allocatable :: r(:) + + irmin=2 + irmax=1 + boxsize=2*irmax+1 + boxsize=3 + ! set up spiral scanning + allocate (iindex(boxsize*boxsize),jindex(boxsize*boxsize),r(boxsize*boxsize)) + + call spiralscan(boxsize,iindex,jindex,r) + + print *,iindex + print *,jindex + print *,r + end + + + subroutine spiralscan(boxsize,iindex,jindex,r) + + integer boxsize,x,y,n,count + integer*4 iindex(boxsize*boxsize),jindex(boxsize*boxsize) + integer*4, allocatable :: array(:,:),scan(:,:),xx(:) + real*4 r(boxsize*boxsize) + + allocate (array(boxsize,boxsize),scan(boxsize,boxsize),xx(boxsize)) + do i=1,boxsize + xx(i)=i-boxsize/2-1 + end do + + x=0 + y=1 + n=0 + count=boxsize; + do i = 1,count + x = x + 1 + array(x,y) = n + n = n + 1 + end do + do + count = count - 1 + do i = 1,count + y = y + 1 + array(x,y) = n + n = n + 1 + end do + do i = 1,count + x = x - 1 + array(x,y) = n + n = n + 1 + end do + if (n > boxsize*boxsize-1) exit + count = count - 1 + do i = 1,count + y = y - 1 + array(x,y) = n + n = n + 1 + end do + do i = 1,count + x = x + 1 + array(x,y) = n + n = n + 1 + end do + if (n > boxsize*boxsize-1) exit + end do + scan=boxsize*boxsize-array + +!c create list of indices in scan order + do i=1,boxsize + do j=1,boxsize + iindex(scan(i,j))=xx(i) + jindex(scan(i,j))=xx(j) + end do + end do + !c the distance between center point and each scanned pixels + do k = 1, boxsize*boxsize + r(k) = sqrt(real(iindex(k))**2.+real(jindex(k))**2.) + end do + + return + end subroutine spiralscan + diff --git a/ps/testspiralscan b/ps/testspiralscan new file mode 100644 index 0000000..6dca638 Binary files /dev/null and b/ps/testspiralscan differ diff --git a/ps/testspiralscan.f90 b/ps/testspiralscan.f90 new file mode 100644 index 0000000..017e5d1 --- /dev/null +++ b/ps/testspiralscan.f90 @@ -0,0 +1,116 @@ +!***** +! +! psiterp algorithm from Wang and Chen translated to f90 +! + + implicit none + + integer boxsize,irmin,irmax,i,nvalid + integer, allocatable :: iindex(:),jindex(:) + real, allocatable :: r(:),r2(:) + + + ! compute the scan indices array + irmin=2 + irmax=3 + boxsize=2*irmax+1 + allocate (iindex(boxsize*boxsize),jindex(boxsize*boxsize),r(boxsize*boxsize)) + call spiralscan(boxsize,iindex,jindex,r,nvalid) + print *,'Spiral indices set' + do i=1,nvalid + print *,i,iindex(i),jindex(i) + end do + +end program + + subroutine spiralscan(boxsize,iindex,jindex,r,nvalid) + + integer boxsize,x,y,n,count,irmin,irmax,p,k,nvalid + integer*4 iindex(boxsize*boxsize),jindex(boxsize*boxsize) + real*4 r(boxsize*boxsize) + integer*1, allocatable :: visited(:,:) + + allocate (visited(boxsize,boxsize)) + + irmin=0 + irmax=boxsize/2 + visited(1,1)=-1 + k=1 + + do i=1,irmax + x=i + y=0 + p=1-i + if(i.gt.irmin)then + iindex(k)=i + jindex(k)=0 + k=k+1 + iindex(k)=-i + jindex(k)=0 + k=k+1 + iindex(k)=0 + jindex(k)=i + k=k+1 + iindex(k)=0 + jindex(k)=-i + k=k+1 + end if + visited(i+1,1)=-1 + visited(1,i+1)=-1 + flag=0 + + do while(x.gt.y) + if(flag.eq.0)then + y=y+1 + if(p.le.0)then + p=p+2*y+1 + else + x=x-1 + p=p+2*y-2*x+1 + end if + else + flag=flag-1 + end if + if(x.lt.y)go to 12 + do while(visited(x-1+1,y+1).eq.0) + x=x-1 + flag=flag+1 + end do + visited(x+1,y+1)=-1 + visited(y+1,x+1)=-1 + if(i.gt.irmin)then + iindex(k)=x + jindex(k)=y + k=k+1 + iindex(k)=-x + jindex(k)=-y + k=k+1 + iindex(k)=x + jindex(k)=-y + k=k+1 + iindex(k)=-x + jindex(k)=y + k=k+1 + if(x.ne.y)then + iindex(k)=y + jindex(k)=x + k=k+1 + iindex(k)=-y + jindex(k)=-x + k=k+1 + iindex(k)=y + jindex(k)=-x + k=k+1 + iindex(k)=-y + jindex(k)=x + k=k+1 + end if + end if + if(flag>0)x=x+1 + end do +12 continue + end do + nvalid=k-1 + return + end subroutine spiralscan + diff --git a/run_kp_sentineltimeseries.py b/run_kp_sentineltimeseries.py new file mode 100644 index 0000000..64ad1bf --- /dev/null +++ b/run_kp_sentineltimeseries.py @@ -0,0 +1,263 @@ +#!/usr/bin/env python3 +# +# +# run_kp_sentineltimeseries.py -- generate subdirectories for a study area and run all available paths given input fileid files +# +# +# INPUTS: + # summary_file: file with study area information (default = './summary_paths' + # upsample_x: factor with which to upsample the 30 m copernicus DEM along x (e.g. 30/upsample_x ~= range resolution) (default=6 --> ~5 m resolution) + # upsample_y: factor with which to upsample the 30 m copernicus DEM along y (e.g. 30/upsample_y ~= azimuth resolution) (default=2 --> ~15 m resolution) + # looksac: number of looks to take in range/longitude (default = 12 --> ~60 m resolution if upsample_x=6) + # looksdn: number of looks to take in azimuth/latitude (default = 4 --> ~60 m resolution if upsample_y=4) + # timebaseline: maximum temporal baseline for sbas (default = 90 days) + # spatialbaseline: maximum spatial baseline for sbas (default = 1000 m) + # TROPO_CORR: 'Y'- apply tropospheric correction (default = 'Y') + # INPUT_UPDATE_FLAG: if you'd like to update everything without even checking (default='N') + # REMOVE_ZIP: remove .zip files for L0 data after the file has been processed/merged (default='Y') + # REMOVE_SAFE: remove .safe files for L0 data after the file has been processed/merged (default='Y') + # REMOVE_ORBTIMING: remove .orbtiming files in RAW directory after the file has been processed/merged (default='Y') + # REMOVE_RAW_GEO: remove .geo files from RAW directory after the file has been merged into its yyyymmdd.geo file in GEO (default='Y') + # WATER_MASK: flag to create a water mask (CURRENTLY NOT FUNCTIONAL) (default, for now = 'N') + # coverage_thresh: threshold for an SLC date coverage within its valid study area (default=30 [%]) + # consistent_thresh: threshold to determine whether the consistent or culled list should be used. If the number of culled(inconsistent) SLCs is greater than the number of consistent SLCs by + # the input consistent_thresh value, then the culled (inconsistent) list is used in further processing. (default = 5) + # LOOK_VECTOR: flag to make look vectors for the multilooked .geo files (necessary when VH-decomposition is functional) (Default='Y') + # ORIGINAL_WORKFLOW: Create a velocity/displacement solution using the original workflow (no phase reconstruction) + # HZ_WORKFLOW: Create a velocity/displacement solution using Howard's phase reconstruction workflow (single similarity mask to define PS pixels) + # KP_WORKFLOW: Create a velocity/displacement solution using Karissa's/Ke's phase reconsutrction workflow (interferogram-specific similarity mask to define PS pixels) + # --> This is currently turned off. The script will work for now, but I already see potential bugs tha will arise as written if using the subscript outside of this script + # ps_thresh: MLE estimator threshold for first pass at finding PS pixels (default=2) + # simthresh: cosine similarity index threshold for second/third passes at finding PS pixels (default=0.5) + # prct_coverage_thresh: when culling .geo dates from final geolist, use similarity masks to determine percentage of valid coverage of PS pixels. If 10%) + # refthresh: threshold used to find reference pixels for tropospheric correction and interferogram calibration for SBAS, based on cosine similarity output. (default = 0.6) + +import os +import getpass +import subprocess +import sys +import time +import copy + +##### ----- SET UP DEFAULT PARAMETERS ----- ##### +sum_file = 'summary_paths' +upsample_x = '6' +upsample_y = '2' +looksac = '12' +looksdn = '4' +timebaseline = '90' +spatialbaseline = '1000' +TROPO_CORR = 'Y' +thresh = '0.5' +INPUT_UPDATE_FLAG = 'N' +REMOVE_ZIP = 'Y' +REMOVE_SAFE = 'Y' +REMOVE_ORBTIMING = 'Y' +REMOVE_RAW_GEO = 'Y' +WATER_MASK = 'N' +coverage_thresh = '30' +consistent_thresh = '5' +LOOK_VECTOR = 'Y' +ORIGINAL_WORKFLOW = 'Y' +HZ_WORKFLOW = 'Y' +KP_WORKFLOW = 'N' +psthresh = '2' +simthresh = '0.5' +prct_coverage_thresh = '0.1' +refthresh = '0.6' + +##### ----- READ INPUTS FILE, IF INDICATED ----- ##### + + + +##### ----- RUN DATA_ARCH.PY TO CHECK/CREATE DEM AND SET UP DATA ARCHITECTURE ----- ##### +print('\n##### ---------- SETTING UP DATA ARCHITECTURE ---------- #####') +command = '$PROC_HOME/kp_scripts/data_arch.py '+sum_file+' '+upsample_x+' '+upsample_y +print(command) +ret = os.system(command) + +# Read Update_Files_Flag +with open('Update_Files_Flag','r') as fflag: + line=fflag.readline().strip() + update_flag = line.split(':')[-1].strip() + +# Check if User wants to overwrite existing files, even if matching DEM exists +if INPUT_UPDATE_FLAG == 'Y': + update_flag = 'Y' + +##### ----- DETERMING THE HOME AREA AND SUBDIRECTORIES TO CYCLE THROUGH ----- ##### +# Open subdirectories and figure out which are available for processing +with open('subdirectories','r') as fsubdirs: + subdirs = [] + for subdir in fsubdirs.readlines(): + if os.path.isfile(subdir.strip()+'/scenelist'): + subdirs.append(subdir.strip()) + +# Get study area directory as 'home' +home = os.getcwd() + +########## ---------- CYCLE THROUGH THE DIRECTORIES FOR PROCESSING ---------- ########## +for subdir in subdirs: + print('\n Moving to '+subdir) + os.chdir(subdir.rstrip()) + if os.path.isdir('RAW') == False: + ret = os.system('mkdir RAW') + os.chdir('RAW') + + ##### ----- DOWNLOAD THE SENTINEL-1 DATA WITH DOWNLOAD_SENTINEL.PY ----- ##### + print('\n##### ----- DOWNLOADING RAW DATA FILES ----- #####') + + command = '$PROC_HOME/kp_scripts/download_sentinel.py ../scenelist '+update_flag + print(' '+command) + ret=os.system(command) + + ##### ----- CREATE GEOCODED SLCS IN EACH FLIGHT PATH DIRECTORY ----- ##### + print('\n##### ---------- GENERATING GEOCODED SLCS ---------- #####') + path_to_geo = subdir+'/GEO' + command = '$PROC_HOME/kp_scripts/generate_geo.py '+home+' '+path_to_geo+' '+update_flag+' '+REMOVE_ZIP+' '+REMOVE_SAFE+' '+REMOVE_ORBTIMING+' '+REMOVE_RAW_GEO + print(' '+command) + ret=os.system(command) + + + ##### ----- GENERATING SLC-LEVEL FILES FOR MULTILOOKED DATA ----- ##### + print('\n##### ----- GENERATING SLC-LEVEL FILES FOR MULTILOOKED DATA ----- #####') + os.chdir(path_to_geo) + ret=os.system('cp '+home+'/params .') + + # Create sbas directory, named by number of looks in each dimension + sbas_path = 'sbas_'+looksac+'_'+looksdn + if os.path.isdir(sbas_path)==False: + command = 'mkdir '+sbas_path + print(' '+command) + ret=os.system(command) + os.chdir(sbas_path) + + # Create Geolist for full-resolution geocoded slcs + command = 'ls ../*.geo |cat> geolist_full' + ret=os.system(command) + ret=os.system('cp '+home+'/params .') + + # Run the script to generate multilooked geocoded slcs, multilooked dem and dem.rsc, study area masks, and look vectors + command = '$PROC_HOME/kp_scripts/ml_scene_level.py geolist_full '+looksac+' '+looksdn+' '+update_flag+' '+WATER_MASK+' '+LOOK_VECTOR + print(' '+command) + ret=os.system(command) + + + ##### ----- GETTING USEFUL PARAMETERS ----- ##### + # Get the Original DEM file size and New DEM file size + with open(home+'/elevation.dem.rsc','r') as fe: + words=fe.readline() + demwidth=words.split()[1].strip() + words = fe.readline() + demlength = words.split()[1].strip() + + with open('dem.rsc','r') as fe: + words=fe.readline() + unwwidth=words.split()[1].strip() + words=fe.readline() + unwlength=words.split()[1].strip() + + + ##### ----- DETERMINE CONSISTENT/INCONSISTENT COVERAGE FROM SCENE MASKS ----- ##### + command = '$PROC_HOME/kp_scripts/determine_consistency.py geolist_ml .. '+coverage_thresh+' '+consistent_thresh + print(' '+command) + ret=os.system(command) + + # Read in CONSISTENT file + with open('CONSISTENT','r') as fcon: + consistent_flag = fcon.readline().strip().split(':')[-1].strip() + culled_flag = fcon.readline().strip().split(':')[-1].strip() + nslc_consistent = fcon.readline().strip().split(':')[-1].strip() + nslc_culled = fcon.readline().strip().split(':')[-1].strip() + + print('\n CULLED_FLAG = '+culled_flag) + + ##### ----- USE DAILY ALL-SIMILARITY MASKS TO FIND & CULL BAD SLCS FROM GEOLIST ----- ##### + print('\n ##### ----- USING COSINE SIMILARITY INDEX TO FIND/CULL BAD SLCs ----- #####') + + if update_flag == 'Y': + print('\n Removing existing interferogram files because update_flag = Y') + command = 'rm *.int *.amp *.cc *.unw' + ret=os.system(command) + + if (update_flag == 'Y') and (os.path.isfile('intlist_'+max_tb)): + with open('intlist_'+max_tb,'r') as fint: + for intname in fint.readlines(): + os.system('rm '+intname.strip()) + + # Determine input parameters for script + if culled_flag == 'Y': + geolistin = 'geolist_culled' + scenemaskin = 'mask_culled' + flag = 'Y' + else: + geolistin = 'geolist_consistent' + scenemaskin = 'mask_consistent' + flag = 'N' + + max_tb = '180' + max_sb = '1000' + + # Run the script + command = '$PROC_HOME/kp_scripts/cull_geolist_fromsim.py '+geolistin+' '+path_to_geo+' '+scenemaskin+' '+looksac+' '+looksdn+' '+flag+' '+max_tb+' '+max_sb+' '+psthresh+' '+simthresh+' '+prct_coverage_thresh + + print('\n Creating final geolist using daily all_similarity_mask') + print(' '+command) + ret=os.system(command) + + # Name some final input parameters + geolistin = geolistin+'_final' + + ##### ----- DOUBLE-CHECK THE FINAL GEOLIST FOR CONSISTENCY ----- ##### + print('\n ##### ----- DOUBLE-CHECK THE FINAL GEOLIST FOR CONSISTENCY ----- #####') + # Double check the final geolist for consistency (just in case inconsistent interferograms were culled from the list) + command = '$PROC_HOME/kp_scripts/determine_consistency.py '+geolistin+' .. '+coverage_thresh+' '+consistent_thresh + print(' '+command) + ret=os.system(command) + + # Read in CONSISTENT file + with open('CONSISTENT','r') as fcon: + consistent_flag = fcon.readline().strip().split(':')[-1].strip() + culled_flag = fcon.readline().strip().split(':')[-1].strip() + nslc_consistent = fcon.readline().strip().split(':')[-1].strip() + nslc_culled = fcon.readline().strip().split(':')[-1].strip() + + # Determine final input parameters for remainder of the script + if culled_flag == 'Y': + geolistin = 'geolist_culled' + scenemaskin = 'mask_culled' + flag = 'Y' + else: + geolistin = 'geolist_consistent' + scenemaskin = 'mask_consistent' + flag = 'N' + + + + ##### ----- RUN ORIGINAL WORKFLOW IF ORIGINAL_WORKFLOW = 'Y' ----- ##### + if ORIGINAL_WORKFLOW == 'Y': + print('\n Running original workflow with '+geolistin) + command = '$PROC_HOME/kp_scripts/original_sbas.py '+geolistin+' '+path_to_geo+' '+timebaseline+' '+spatialbaseline+' '+looksac+' '+looksdn+' '+scenemaskin+' '+flag+' '+psthresh+' '+simthresh+' '+refthresh+' '+TROPO_CORR + print(' '+command) + ret=os.system(command) + + + ##### ----- RUN HOWARD'S PHASE RECONSTRUCTION WORKFLOW IF HZ_WORFKLOW = 'Y' ----- ##### + if HZ_WORKFLOW == 'Y': + print('\n Running phase reconstruction (single mask) & SBAS with '+geolistin) + command = '$PROC_HOME/kp_scripts/PR_sbas_singlemask.py hz_ints '+geolistin+' '+path_to_geo+' '+timebaseline+' '+spatialbaseline+' '+looksac+' '+looksdn+' '+scenemaskin+' '+flag+' '+psthresh+' '+simthresh+' '+refthresh+' '+TROPO_CORR + print(' '+command) + ret=os.system(command) + + +''' + ##### ----- RUN KARISSA'S ALIASING IDENTIFICATION AND PHASE RECONSTRUCTION WORKFLOW IF KP_WORKFLOW = 'Y' ----- ##### + if KP_WORKFLOW == 'Y': + print('\n Running phase reconstruction (interferogram-specific mask) & SBAS with '+geolistin) + command = '$PROC_HOME/kp_scripts/PR_sbas_multimask.py kp_ints '+geolistin+' '+path_to_geo+' '+timebaseline+' '+spatialbaseline+' '+looksac+' '+looksdn+' '+scenemaskin+' '+flag+' '+psthresh+' '+simthresh+' '+refthresh+' '+TROPO_CORR + print(' '+command) + ret=os.system(command) +''' + + diff --git a/sbas/.nfs000000000001a90100000001 b/sbas/.nfs000000000001a90100000001 new file mode 100644 index 0000000..b16debd Binary files /dev/null and b/sbas/.nfs000000000001a90100000001 differ diff --git a/sbas/sbas b/sbas/sbas new file mode 100644 index 0000000..dbf79c0 Binary files /dev/null and b/sbas/sbas differ diff --git a/sentinel/azimuth_compress.cu b/sentinel/azimuth_compress.cu index 68e141e..f88b968 100644 --- a/sentinel/azimuth_compress.cu +++ b/sentinel/azimuth_compress.cu @@ -258,10 +258,10 @@ extern "C" void azimuth_compress_( long int getgpu = cudaSetDevice(0); // printf("GPU set return: %d\n",getgpu); if (getgpu != 0){ - printf("Can't grab GPU %ld\n",getgpu); + printf("Can't grab GPU %d\n",getgpu); FILE *fgetgpu = NULL; fgetgpu =fopen("getgpulog","a"); - fprintf(fgetgpu,"Can't grab GPU %ld\n",getgpu); + fprintf(fgetgpu,"Can't grab GPU %d\n",getgpu); fclose(fgetgpu); } @@ -402,7 +402,7 @@ lon = (double *) malloc(*demwidth * sizeof(double)); iaddr_size_t= (long long int) nlines * (long long int) *demwidth * (long long int) 2; nbytes=read(*fddem,demin,iaddr_size_t); if (nbytes < 0) { - printf("dem read error %lld\n",nbytes); + printf("dem read error %d\n",nbytes); printf("iaddr_off_t %ld, iaddr_size_t %ld\n",iaddr_off_t, iaddr_size_t); printf("nlines %d, demwidth %d\n",nlines,*demwidth); } diff --git a/sentinel/azimuth_compress.o b/sentinel/azimuth_compress.o new file mode 100644 index 0000000..bd46b32 Binary files /dev/null and b/sentinel/azimuth_compress.o differ diff --git a/sentinel/azimuth_compress_cpu.o b/sentinel/azimuth_compress_cpu.o new file mode 100644 index 0000000..5fe3031 Binary files /dev/null and b/sentinel/azimuth_compress_cpu.o differ diff --git a/sentinel/azimuth_compress_cpu_losvec.c b/sentinel/azimuth_compress_cpu_losvec.c new file mode 100644 index 0000000..087fba0 --- /dev/null +++ b/sentinel/azimuth_compress_cpu_losvec.c @@ -0,0 +1,419 @@ +//!!!!!!!!!!!!!!! +//! +//! +//! azimuth compression subroutine for use in Sentinel back projection processor +//! called from a fortran main program +//! +//! +//!!!!!!!!!!!!!! + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/** + * Find maximum between two numbers. + */ +int max(int num1, int num2) +{ + return (num1 > num2 ) ? num1 : num2; +} + +/** + * Find minimum between two numbers. + */ +int min(int num1, int num2) +{ + return (num1 > num2 ) ? num2 : num1; +} + +extern void orbitrangetime_(double *xyz,double *timeorbit,double *xx, double *vv,int *numstatevec,double *tmid,double *satx, double *satv,double *tline,double *rngpix); + +// pixelint - the routine to compute complex sum in cpu + +void pixelint(_Complex float *burstdata_d, double *satloc_d, double *xyz_d, double *azoff_d, int demwidth_d, int aperture_d, int iaperture_d, int rawdatalines_d, int samplesPerBurst_d, double rngstart_d, double rngend_d, double dmrg_d, double wvl_d, _Complex float *outdata_d, int nlines_d){ + + // internal variables for integration + _Complex double cacc, cphase; + _Complex float cval; + int intr; + double range, fracr,phase,r,vectx,vecty,vectz,pi; + int azline; + int azstart, azend; + long loop; + + pi=4.*atan2(1.,1.); + + // loop over 1D version of 2D arrays +#pragma omp parallel for shared(nlines_d, demwidth_d, azoff_d, aperture_d, rawdatalines_d, xyz_d, satloc_d, rngstart_d, rngend_d, dmrg_d, burstdata_d, samplesPerBurst_d, pi, wvl_d, outdata_d) private(cacc, azstart, azend, azline, vectx, vecty, vectz, range, r, intr, fracr, cval, phase, cphase) + for (loop = 0; loop < nlines_d * demwidth_d; loop++){ + + cacc = 0. + 0. * I; + if (azoff_d[loop] > 0.){ + azstart=round(azoff_d[loop] - aperture_d / 2); + azend=round(azoff_d[loop] + aperture_d / 2); + azstart=max(azstart,0); + azend=min(azend,rawdatalines_d-1); + for (azline=azstart; azline=rngstart_d && range <= rngend_d - dmrg_d){ + r=(range - rngstart_d) / dmrg_d ; + intr=floor(r); + fracr=r-intr; + + cval=burstdata_d[azline * samplesPerBurst_d + intr]*(1-fracr)+burstdata_d[azline * samplesPerBurst_d + (intr+1)]*fracr; + + phase = 4. * pi / wvl_d * range; + cphase = cos(phase) + sin(phase) * I; + cacc = cacc+cval*cphase; + } // end range test + } // end azline loop + } // end pixel test + + outdata_d[loop]=cacc; + + } // end loop loop and parallel section + +} + +// set xyz array +void setxyz(short *demin_d, double *xyz_d, double *xyzfit_d, double *azoff_d, int demwidth_d, double firstlat_d, double deltalat_d, double firstlon_d, double deltalon_d, int firstline_d, int nlines_d, int firstpix_d, int lastpix_d){ + + // local array definitions for loop + double *llh = (double *) malloc(sizeof(double)*3); + double *xyztemp = (double *) malloc(sizeof(double)*3); + double *satx = (double *) malloc(sizeof(double)*3); + double *satv = (double *) malloc(sizeof(double)*3); + double *unitlookvector = (double *) malloc(sizeof(double)*3); + + double pi = 4.0*atan2(1.,1.); + double deg2rad = pi / 180.; + double a = 6378137.0; + double e2 = 0.0066943799901499996; + double re; + int line, pixel; + double lat; + long loop; + long long xyzoffset; + double llhlat, llhlon, llhhgt; + + // loop over full array +#pragma omp parallel for shared(demwidth_d, nlines_d, firstlat_d, firstline_d, deltalat_d, firstpix_d, lastpix_d, deg2rad, firstlon_d, deltalon_d, demin_d, a, e2, xyzfit_d, xyz_d) private(line, pixel, lat, xyzoffset, llhlat, llhlon, llhhgt, re) + for (loop = 0; loop < demwidth_d * nlines_d; loop++){ + line = (int) (loop / demwidth_d); + pixel = loop - line * demwidth_d; + lat = firstlat_d + (line + firstline_d) * deltalat_d; + xyzoffset = ((long long int) line * (long long int) demwidth_d + (long long int) pixel ) * (long long int) 3; + xyz_d[xyzoffset+0]=0.; + xyz_d[xyzoffset+1]=0.; + xyz_d[xyzoffset+2]=0.; + + if (pixel >= firstpix_d && pixel <= lastpix_d){ + + + + llhlat = lat * deg2rad; + llhlon = (firstlon_d + pixel * deltalon_d) * deg2rad; + llhhgt = demin_d[xyzoffset / 3]; + + re = a/sqrt(1.0 - e2*sin(llhlat)*sin(llhlat)); + + xyz_d[xyzoffset+0] = (re + llhhgt)*cos(llhlat)*cos(llhlon); + xyz_d[xyzoffset+1] = (re + llhhgt)*cos(llhlat)*sin(llhlon); + xyz_d[xyzoffset+2] = (re - (re * e2) + llhhgt)*sin(llhlat); + + if(pixel == firstpix_d){ + xyzfit_d[line * 9 + 0]=xyz_d[xyzoffset+0]; + xyzfit_d[line * 9 + 1]=xyz_d[xyzoffset+1]; + xyzfit_d[line * 9 + 2]=xyz_d[xyzoffset+2]; + } + if(pixel == firstpix_d + (int)((lastpix_d-firstpix_d)/2-1)){ + xyzfit_d[line * 9 + 3]=xyz_d[xyzoffset+0]; + xyzfit_d[line * 9 + 4]=xyz_d[xyzoffset+1]; + xyzfit_d[line * 9 + 5]=xyz_d[xyzoffset+2]; + } + if(pixel == firstpix_d + 2 * (int)((lastpix_d-firstpix_d)/2-1)){ + xyzfit_d[line * 9 + 6]=xyz_d[xyzoffset+0]; + xyzfit_d[line * 9 + 7]=xyz_d[xyzoffset+1]; + xyzfit_d[line * 9 + 8]=xyz_d[xyzoffset+2]; + } + + } // end pixel test + } // end loop loop and parallel section +} // end routine setxyz + +// set azoff array +void setazoff(double *coef_d, double *azoff_d, int demwidth_d, int nlines_d, int firstpix_d, int lastpix_d, int aperture_d, int rawdatalines_d){ + + int line, pixel; + double fit, arg; + long loop; + + for (loop = 0; loop < demwidth_d * nlines_d; loop++){ + line = (int) (loop / demwidth_d); + pixel = loop - line * demwidth_d; +// printf("line pixel %d %d\n",line, pixel); + + if (pixel >= firstpix_d && pixel <= lastpix_d){ + arg=((float)(pixel-firstpix_d)/(float)(lastpix_d-firstpix_d)*2.-1.); + fit=coef_d[line*3+0]*arg*arg+coef_d[line*3+1]*arg+coef_d[line*3+2]; + azoff_d[line * demwidth_d + pixel]=-1; + if (fit > aperture_d/2 && fit < rawdatalines_d- aperture_d/2){ + azoff_d[line * demwidth_d + pixel]=fit; + } // end fit in aperture test + } // end pixel test + } // end loop loop +} //end setazoff routine + + +extern void azimuth_compress_cpu_( + float complex *burstdata, + double *satloc, + int *rawdatalines, + int *samplesPerBurst, + int *demwidth, + int *demlength, + int *fdout, + int *fddem, + int *fdlos, + double *deltalat, + double *deltalon, + double *firstlat, + double *firstlon, + double *latlons, + double *timeorbit, + double *xx, + double *vv, + int *numstatevec, + double *rngstart, + double *rngend, + double *tstart, + double *tend, + double *tmid, + double *xyz_mid, + double *vel_mid, + double *t, + double *dtaz, + double *dmrg, + double *wvl, + int *aperture, + int *iaperture, + double *angc0, + double *angc1, + double *prf) +{ + + // internal variables + double *lon; + short *demin; + double *xyz, *azoff; + double tline; + double rngpix; + double *xyzfit; + double *coef; + double umag, fd, veff, td, udotv; + int firstline, lastline; // limits on line loop + int firstpix, lastpix; // limits on pixel loop + long int arraysize; + FILE *fpout; // stream for file descriptor *fdout + FILE *fplos; // stream for the file descriptor *fdlos + + int naperture; // naperture is integration midpoint in pixels + int y1,y2,y3; + int pixel,line,i,j,ipix; + int nbytes; + int nlines; + off_t iaddr_off_t; + size_t iaddr_size_t; + float complex *outdata, *indata; + + // variables for openmp pragma argument passing + double omptmid=*tmid; + double ompangc0=*angc0, ompangc1=*angc1; + double ompprf=*prf; + int omprawdatalines=*rawdatalines; + double ompwvl=*wvl; + + // how much memory will we need to allocate? + //printf("latlons %f %f %f %f\n",latlons[0],latlons[1],latlons[2],latlons[3]); + lastline=(int)((latlons[0] - *firstlat) / *deltalat ); + firstline=(int)((latlons[1] - *firstlat) / *deltalat ); + if (firstline < 0) firstline=0; + if (lastline < 0) lastline=0; + if (firstline > *demlength-1) firstline= *demlength -1; + if (lastline > *demlength-1) lastline= *demlength -1; + nlines=lastline-firstline+1; + printf("Burst line limits, size (first,last,nlines) %d %d %d\n",firstline,lastline,nlines); + if (nlines == 1){ + printf("Burst not in DEM, skipped.\n"); + return; + } + // malloc cpu arrays + arraysize = (long int) nlines * (long int) *demwidth; + lon = (double *) malloc(*demwidth * sizeof(double)); + demin = (short *) malloc(arraysize * sizeof(short)); + azoff = (double *) malloc(arraysize * sizeof(double)); + xyz = (double *) malloc(arraysize * sizeof(double) * 3); + xyzfit = (double *) malloc(nlines * sizeof(double) * 9); + coef = (double *) malloc(nlines * sizeof(double) * 3); + outdata = (float complex *)malloc(arraysize * sizeof(float complex)); + indata = (float complex *)malloc(arraysize * sizeof(float complex)); + outlos = (double *) malloc(arraysize * sizeof(double) * 3); + inlos = (double *) malloc(arraysize * sizeof(double) * 3); + + // cpu array definitions + // _Complex float *burstdata_d; + double *satloc_d; + double *azoff_d, *xyz_d; + double *xyzfit_d; + double *coef_d; + // _Complex float *outdata_d; + short *demin_d; + + // constants and such + double pi; + pi = 4. * atan2(1.0,1.0); + + // set up longitude loop + firstpix=(latlons[2]-*firstlon)/ *deltalon; if (firstpix < 0)firstpix=0; + lastpix=(latlons[3]-*firstlon)/ *deltalon; if (lastpix > *demwidth)lastpix= *demwidth; + + // define longitude array + for (i=0;i<*demwidth;i++){ + lon[i]=*firstlon + (i-1)* *deltalon; + } + + // zero out data array before integration + for (j=0; j 1.e-18)indata[line * *demwidth + pixel]=outdata[line * *demwidth + pixel]; + } + } + // write line to file + nbytes=fseek(fpout, iaddr_off_t, SEEK_SET); + nbytes=fwrite(indata, 1, iaddr_size_t, fpout); + + // KARISSA, THIS IS WHERE TO WRITE OUT THE LOS VECTOR??? + // what is the format for writing the file? what are these nbytes? what is this stuff? iaddr_off_t, SEEK_SET, iaddr_size_t, indata? + // Ideally I would write the los vector here? + // fplos=fdopen(*fdlos,"w+") + // nbytes=frwite(what? and where?) + + + // free up cpu memory + free(indata); + free(outdata); + free(demin); + free(xyzfit); + free(coef); + free(lon); + free(xyz); + free(azoff); +} + diff --git a/sentinel/backprojectcpusub.o b/sentinel/backprojectcpusub.o new file mode 100644 index 0000000..4c2d59b Binary files /dev/null and b/sentinel/backprojectcpusub.o differ diff --git a/sentinel/backprojectcpusub_losvec.f90 b/sentinel/backprojectcpusub_losvec.f90 new file mode 100644 index 0000000..1f02fc8 --- /dev/null +++ b/sentinel/backprojectcpusub_losvec.f90 @@ -0,0 +1,445 @@ +!!!!! +! +! backprojectsub_losvec - subroutine to back project a sentinel burst to a regular grid defined by dem file +! use gpu for complex integration, and also writes an LOS file +! +! steps: +! 1. Inputs- get metadata, dem info, statevectors, sc position array +! 2. Loop over all bursts for subswath +! 3. Extract burst specific parameters from this bursts database +! 4. Read in range compressed data +! 5. Back project to each point in the dem and write the los_vec file +! +!!!!! + +subroutine backprojectcpusub_losvec(rangeprocdata, rangesamples, linestotal, nbursts, lines, slcoutfile, SAFEname, swath, losoutfile) + +! use sql_mod + implicit none + + ! arguments + complex rangeprocdata(rangesamples,linestotal) + integer rangesamples, nbursts, linestotal, lines(nbursts), swath + character*300 slcoutfile + character*200 SAFEname + character*300 losoutfile + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! DECLARE LOCAL VARIABLES +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + character*300 str,rawinfile,dbfile,demfile,demrscfile,burstfile + character*300 orbtimingfile,units,type,table,posfile + + integer*1 header(80) + integer stat,intp_orbit,isafe + integer*4 azimuthBursts,samplesPerBurst + integer*4 fddem, fdout, initdk + integer*4 demwidth,demlength,idemwidth,idemlength,width,length + integer*4 burst,nlines,numstatevec,iaperture,linesmax, ioffset + integer*8 db + real*8 pri, range0, samplefreq, starttime + real*8 ptsperdempt, posting, deltalat,deltalon,firstlat,firstlon + real*8 timeorbit(1000),xx(3,1000),vv(3,1000),aa(3,1000),x(3),v(3),a(3) + real*8 timefirst,timeend,slantRangeTime,rangeSamplingRate,prf,startime(1000) + real*8 satloc(3,100000),satvel(3,100000) + real*8 :: lat, fd, ht, re, vmag + real*8 :: azoff, fdcoefs(2) + +!!!!Image limits + real*8 tstart, tend, tline + real*8 rngstart, rngend, rngpix, latlons(4), maxlatlon + +!!!! Satellite positions + real*8, dimension(3) :: xyz_mid, vel_mid, acc_mid + real*8 :: tmid, rngmid, temp, t(100000) + + real*8 :: llh(3),xyz(3) + real*8 :: satx(3), satv(3),sata(3) + integer :: pixel,line,ith,i + + integer :: i_type, azline, intr,aperture + real*8 :: dtaz, dmrg, range,r,fracr,phase + complex*8 :: val + complex*16 :: cacc + complex*8 :: pixelint + + real*4 :: timer0, timer1 + ! array to hold each burst + complex*8, allocatable :: burstdata(:,:) + real*8 :: wvl, r0 + + integer*8 filelen + integer*4 rawdatalines,nbytes + integer*4 iburst, j, irec + real*8 azimuthTimeInterval,radarFrequency + real*8 timecenterseconds,rawdataprf + + ! tops params + real*8 ks,apertureground,aperturefd,aperturetime,d,bursttimespan + integer burstpoints, naperture, napertureorig + real*8 unitlookvector(3),td,fdc, frate, angc0, angc1 + + ! declare some constants + integer LLH_2_XYZ + real*8 pi,rad2deg,deg2rad,sol + real*4 BAD_VALUE + parameter(BAD_VALUE = -999999.0) + parameter(LLH_2_XYZ=1) + + !c types needed + + type :: ellipsoid + real*8 r_a ! semi-major axis + real*8 r_e2 ! eccentricity of earth ellisoid + end type ellipsoid + type(ellipsoid) :: elp + + elp%r_a=6378137.0 + elp%r_e2=0.0066943799901499996 + + pi = 4.d0*atan2(1.d0,1.d0) + sol = 299792458.d0 + rad2deg = 180.d0/pi + deg2rad = pi/180.d0 + + !c get dem and rsc file names + open(21,file='params') + read(21,'(a)')demfile + read(21,'(a)')demrscfile + close(21) +!!$ print *,'dem file: ',demfile +!!$ print *,'demrscfile: ',demrscfile + + !c read in the dem and its resource parameters + open(21,file=demrscfile) + read(21,'(a)')str + read(str(15:60),*)demwidth + read(21,'(a)')str + read(str(15:60),*)demlength + read(21,'(a)')str + read(str(15:60),*)firstlon + read(21,'(a)')str + read(str(15:60),*)firstlat + read(21,'(a)')str + read(str(15:60),*)deltalon + read(21,'(a)')str + read(str(15:60),*)deltalat + close(21) + + !c open the dem file + fddem=initdk(31,demfile) + + !c loop over all bursts in this subswath + linesmax=linestotal/nbursts + !print *,'linesmax ',linesmax, nbursts + +! SAFEname to point to correct positionfile + do isafe=1,len_trim(SAFEname) + if (ichar(SAFEname(isafe:isafe)).eq.0)exit + end do + + do iburst=1,nbursts + header=transfer(rangeprocdata(1:10,1+linesmax*(iburst-1)),header) + call burstparams(header, starttime, prf, samplefreq, pri, range0, wvl) +! call burstparams(rangeprocdata(1,1+linesmax*(iburst-1)), starttime, prf, samplefreq, pri, range0, wvl) + !print *,'burstparams ',starttime,prf,samplefreq,pri,range0,wvl + slantRangeTime=2*range0/sol + if(iburst.le.9)posfile=SAFEname(1:isafe-1)//'.positionburst'//char(iburst+48)//char(48+swath)//'.out' + if(iburst.ge.10)posfile=SAFEname(1:isafe-1)//'.positionburst1'//char(iburst-10+48)//char(48+swath)//'.out' +!if(iburst.le.9)posfile='positionburst'//char(iburst+48)//'.out' +! if(iburst.ge.10)posfile='positionburst1'//char(iburst-10+48)//'.out' +! print *,'accessing positionfile ',posfile,lines + re=6378137.d0 + ht=700000.d0 + rawdatalines=lines(iburst) +! print *,'raw data lines ', rawdatalines + + !c read in the orbit state vectors + + open(21,file=SAFEname(1:isafe-1)//'.orbtiming') !orbtimingfile) + read(21,*)timefirst + read(21,*)timeend + read(21,*)nlines + read(21,*)numstatevec + !print *,'Number of state vectors: ',numstatevec + + !c read in state vectors + do i=1,numstatevec + read(21,*)timeorbit(i),x,v,a + xx(:,i)=x + vv(:,i)=v + aa(:,i)=a + !print *,timeorbit(i) + end do + close(21) + + !c read in position file +! print *,'position file ',posfile + open(21,file=posfile) + do i=1,rawdatalines + read(21,*,end=102)line,t(i),satloc(:,i),satvel(:,i) + end do +102 close(21) + + ! get starting time + timer0 = secnds(0.0) + + ! estimate doppler centroid + call fdopcoefs(rangeprocdata(1,1+linesmax*(iburst-1)),rangesamples,rawdatalines,fdcoefs) + angc0=fdcoefs(1) + angc1=fdcoefs(2) +! print *,'fd coefs ',angc0, angc1 + + rangeprocdata(1:10,1+linesmax*(iburst-1):linesmax*iburst)=cmplx(0.,0.) + + !c set the size of the aperture and its centroid (10 m antenna) + v=satvel(:,rawdatalines/2) + vmag=dsqrt(dot_product(v,v)) + vmag=vmag*(re/(re+ht)) + dmrg = sol/2.d0/rangeSamplingRate + temp=fd*prf*prf*wvl*(range0+rangesamples/2.*dmrg)/2.d0/vmag**2 + iaperture=temp + !print *,'aperture pts, offset: ',aperture,iaperture,temp + + ! tops geometry params + d=10. + apertureground=range0*wvl/d + bursttimespan=rawdatalines/prf + + ! refined numbers for tops using both ks and velocity + aperturetime=(range0*wvl/2/vmag/vmag*prf)/(1.+range0*angc1*pi/180./vmag) + burstpoints=aperturetime*prf*0.5 ! 50% of nominal antenna aperture + aperture=burstpoints + +!c open the output file + fdout=initdk(30,slcoutfile) + +!c open the output los vector file + fdlos=initdk(32,losoutfile) + + !c azimuth and range limits + tstart = starttime + dtaz = 1.d0 / prf + tend = tstart + (rawdatalines-1)* dtaz + tmid = 0.5d0*(tstart+tend) + + !print *, 'Start, stop Acquisition time: ', tstart,tend + + rngstart = slantRangeTime*sol/2.d0 + dmrg = sol/2.d0/samplefreq + rngend = rngstart + (rangesamples-1)*dmrg + rngmid = 0.5d0*(rngstart+rngend) + + ! geolocation of the current burst entended for TOPS scan + call bounds(tstart-2.,tend+2.,rngstart,rngend,timeorbit,xx,vv,numstatevec,latlons) + +!!!!Initialize satellite position + tline = tmid + stat = intp_orbit(timeorbit, xx, vv, numstatevec, tline, xyz_mid, vel_mid) + + if (stat.ne.0) then + print *, 'Cannot interpolate orbits at the center of scene.' + stop + endif + + ! azimuth compress + ioffset = linesmax * (iburst-1) +1 + call azimuth_compress_cpu_losvec(rangeprocdata(1,ioffset),satloc,rawdatalines,rangesamples,demwidth,demlength,fdout,fddem,fdlos, & + deltalat,deltalon,firstlat,firstlon, & + latlons,timeorbit,xx,vv,numstatevec,rngstart,rngend,tstart,tend, & + tmid,xyz_mid,vel_mid,t,dtaz,dmrg,wvl,aperture,iaperture,angc0,angc1,prf) + +end do ! end bursts loop + +return +end subroutine backprojectcpusub_losvec + +! fdop - subroutine to get burst angle steering coefficients +! +! derive from commanded azimuth beam position +! +! from burst_angle_steering.f90: get steering angle coefficients for a burst +! + +subroutine fdopcoefs(data,len,lines,fdcoefs) + + complex data(len,lines) + real*8 fdcoefs(2) + + integer word, elevationbeam, azimuthbeam, i + integer*1 byte + integer*1, allocatable :: b(:,:) + complex, allocatable :: header(:,:) + real, allocatable :: t(:), ang(:), prf(:) + real*8 fref + + fref=37.53472224 + + allocate (b(80,lines), header(10,lines)) + allocate (t(lines),ang(lines),prf(lines)) + !print *,'arrays allocated',len,lines,data(100:103,100:103) + + ! save headers + header=data(1:10,:) + do i=1,lines + b(:,i)=transfer(header(:,i),b) + end do + !call headerbytes(header,b,lines) + !b=transfer(header,byte) + + ! get azimuth beam address +! open(31,file='azibeam.out') + do i=1,lines + word=in2(b(1+60,i)) + elevationbeam=iand(ishft(word,-12),15) + azimuthbeam=iand(word,1023) + ang(i)=azimuthbeam/1024.*1.8-0.9 + prf(i)=fref/in3(b(1+50,i))*1.e6 + t(i)=(i-lines/2.)/prf(i) +! write(31,*)i,azimuthbeam,ang(i),t(i) + end do +! close(31) + +! solve for angle as function of time + call linfit(t(21),ang(21),lines-40,angc0,angc1) + !print *,'Angle coeffs: ',angc0,angc1 + !print *,angc0,angc1 + fdcoefs(1)=angc0 + fdcoefs(2)=angc1 + +return + +end subroutine fdopcoefs + + +subroutine medfilt(fd,len,n) + + real, allocatable :: filt(:),temp(:) + real fd(len) + + allocate (filt(len),temp(n)) + + do i=1,n/2 + filt(i)=0. + filt(len+1-i)=0. + end do + do i=n/2+1,len-n/2 + temp=fd(i-n/2:i+n/2) + call sort(n,temp) + filt(i)=temp(n/2+1) + end do + fd=filt + return +end subroutine medfilt + + SUBROUTINE SORT(N,RA) + DIMENSION RA(N) + L=N/2+1 + IR=N +10 CONTINUE + IF(L.GT.1)THEN + L=L-1 + RRA=RA(L) + ELSE + RRA=RA(IR) + RA(IR)=RA(1) + IR=IR-1 + IF(IR.EQ.1)THEN + RA(1)=RRA + RETURN + ENDIF + ENDIF + I=L + J=L+L +20 IF(J.LE.IR)THEN + IF(J.LT.IR)THEN + IF(RA(J).LT.RA(J+1))J=J+1 + ENDIF + IF(RRA.LT.RA(J))THEN + RA(I)=RA(J) + I=J + J=J+J + ELSE + J=IR+1 + ENDIF + GO TO 20 + ENDIF + RA(I)=RRA + GO TO 10 + END + +subroutine linfit(xdata,ydata,npts,c0,c1) + + implicit none + real*8 sumx,sumx2,sumxy,sumy,sumy2 + real xdata(npts),ydata(npts),c0,c1 + integer npts,i + + sumx=0. + sumx2=0. + sumxy=0. + sumy=0. + sumy2=0. + do i=1,npts + !print *,i,xdata(i),ydata(i) + sumx = sumx + xdata(i) + sumx2 = sumx2 + xdata(i) * xdata(i) + sumxy = sumxy + xdata(i) * ydata(i) + sumy = sumy + ydata(i) + sumy2 = sumy2 + ydata(i) * ydata(i) + end do + + c1 = (npts * sumxy - sumx * sumy) / (npts * sumx2 - sumx**2) + c0 = (sumy * sumx2 - sumx * sumxy) / (npts * sumx2 - sumx**2) + return + + end subroutine linfit + +subroutine headerbytes(in, out, lines) + + integer*1 in(80,lines), out(80,lines) + integer lines,i + + out=in + + return +end subroutine headerbytes + +! get some parameters from a burst needed for processing +subroutine burstparams(rawdata, starttime, prf, samplefreq, pri, range0, wvl) + + integer*1 rawdata(80) + real*8 starttime, prf, samplefreq, pri, range0 + real*8 fref, c, wvl + real*8 swst + real*8 d8 + real*8 samplefrequency + integer irank, rangedecimation + + fref=37.53472224d0 + c=299792458.d0 + wvl=0.05546576 + starttime = d8(rawdata(1+68)) + prf = fref/in3(rawdata(1+50))*1.e6 + rangedecimation = iand(int(rawdata(1+40)), 255) + samplefreq = samplefrequency(rangedecimation)*1.e6 + swst = in3(rawdata(1+53))/fref*1.e-6; + irank=iand(int(rawdata(1+49)), 31) + pri = in3(rawdata(1+50))/fref*1.e-6; + range0=c/2.*(irank*pri+swst)+60.; ! add 60 m calibration factor + + return +end subroutine burstparams + +real*8 function d8(data) + integer*1 data(*) + integer*1 b(8) + real*8 d + equivalence (b,d) + + b=data(1:8) + d8=d + !print *,b,d + return + end + diff --git a/sentinel/backprojectgpusub.o b/sentinel/backprojectgpusub.o new file mode 100644 index 0000000..347c07f Binary files /dev/null and b/sentinel/backprojectgpusub.o differ diff --git a/sentinel/bounds.o b/sentinel/bounds.o new file mode 100644 index 0000000..624a45f Binary files /dev/null and b/sentinel/bounds.o differ diff --git a/sentinel/combine_sbas_lists.py b/sentinel/combine_sbas_lists.py new file mode 100644 index 0000000..d0a8bd7 --- /dev/null +++ b/sentinel/combine_sbas_lists.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 + +# combine multiple sbas lists to find unique union of all of them + +import sys +import subprocess +from datetime import datetime +import os + +if len(sys.argv) < 3: + print('Usage: combine_sbas_lists.py listofsbas outfile') + sys.exit(1) + +listofsbas = sys.argv[1] +outfile = sys.argv[2] + +# open listofsbas +fsbas = open(listofsbas,'r') +sbaslists = fsbas.readlines() +fsbas.close() + +# Make sbas lists +sbas = [] +for filename in sbaslists: + print(filename.strip()) + # nearest neighbor + fsbas = open(filename.strip(),'r') + sbas_list = fsbas.readlines() + for line in sbas_list: + if line not in sbas: + sbas.append(line) + +fout = open(outfile,'w') +for line in sbas: + fout.write(line) +fout.close() + +print ('#### ---- FULL SBAS LISTS WRITTEN ----- #####') diff --git a/sentinel/createslc b/sentinel/createslc new file mode 100644 index 0000000..bb63c59 Binary files /dev/null and b/sentinel/createslc differ diff --git a/sentinel/cross.o b/sentinel/cross.o new file mode 100644 index 0000000..bf6fb44 Binary files /dev/null and b/sentinel/cross.o differ diff --git a/sentinel/curvature.o b/sentinel/curvature.o new file mode 100644 index 0000000..544ec9e Binary files /dev/null and b/sentinel/curvature.o differ diff --git a/sentinel/decode_line_memory.o b/sentinel/decode_line_memory.o new file mode 100644 index 0000000..fb93861 Binary files /dev/null and b/sentinel/decode_line_memory.o differ diff --git a/sentinel/filelen.o b/sentinel/filelen.o new file mode 100644 index 0000000..11d00db Binary files /dev/null and b/sentinel/filelen.o differ diff --git a/sentinel/geo2rdr/estimatebaseline b/sentinel/geo2rdr/estimatebaseline new file mode 100644 index 0000000..4f126a9 Binary files /dev/null and b/sentinel/geo2rdr/estimatebaseline differ diff --git a/sentinel/getburststatevectors.o b/sentinel/getburststatevectors.o new file mode 100644 index 0000000..6d9aa35 Binary files /dev/null and b/sentinel/getburststatevectors.o differ diff --git a/sentinel/howmanygpus b/sentinel/howmanygpus new file mode 100644 index 0000000..760d8d0 Binary files /dev/null and b/sentinel/howmanygpus differ diff --git a/sentinel/intp_orbit.o b/sentinel/intp_orbit.o new file mode 100644 index 0000000..d349b3a Binary files /dev/null and b/sentinel/intp_orbit.o differ diff --git a/sentinel/io.o b/sentinel/io.o new file mode 100644 index 0000000..2c3a311 Binary files /dev/null and b/sentinel/io.o differ diff --git a/sentinel/latlon.o b/sentinel/latlon.o new file mode 100644 index 0000000..6d7c60a Binary files /dev/null and b/sentinel/latlon.o differ diff --git a/sentinel/lookvector b/sentinel/lookvector new file mode 100644 index 0000000..3d70fb3 Binary files /dev/null and b/sentinel/lookvector differ diff --git a/sentinel/lookvector.f90 b/sentinel/lookvector.f90 new file mode 100644 index 0000000..8e6f54e --- /dev/null +++ b/sentinel/lookvector.f90 @@ -0,0 +1,344 @@ +program lookvector + ! compute an 'image' of the look vector to each pixel + ! output image is e,n,u components of vector from ground to satellite + + use omp_lib + + implicit none + real*8, allocatable :: timeorbit(:),xx(:,:),vv(:,:) + real*8 timedelta,x(3),v(3),xxmid(3),vvmid(3),timemid + real*8 startime,endtime,startr,endr,latlons(4) + real*8 xyzsatstart(3),velsatstart(3),xyzsatend(3),velsatend(3) + real*8 xyzsatmid(3),velsatmid(3),xyz(3),xyz0(3),xyz1(3),xyzvel(3),uvenu(3) + real*8 llhsat(3),llh(3),xyzsat(3),velsat(3),time + real*8 vhat(3),that(3),nhat(3),chat(3) + real*8 alpha,beta,gamm,delta(3),aa,bb,hgts(2),zsch,costheta,sintheta,dopfact + real*8 pi + real*8 r_lati,r_loni,r_latf,r_lonf,r_geohdg,geohdg + real*8 rcurv,rng,r_e2,r_a,vmag,height + real*8 min_lat,max_lat,min_lon,max_lon + real*8 firstlon, firstlat, deltalon, deltalat + real*8 r,cosalpha,cosalphaprime,deltar,xspace,yspace + real*8 r_p, r_local, rnum, rden, rclose + real*8 uxyz(3),uenu(3),ue(3),un(3),uu(3),rlook(3),llhtest(3),uproj(3),llhmidpoint(3) + real*8 llhtouse(3) + real*8, allocatable :: lookvec(:,:) + integer demwidth, demlength,status,buff(13),k,kk + integer i_type,ind,iter,line,numstatevec,i,j,ret,iclose,jclose,nvect,ilocation + character*300 posfile, str + real*8 q,qq + + ! function types + real*8 norm2 + integer intp_orbit + + integer unit + + !c types needed + type :: ellipsoid + real*8 r_a ! semi-major axis + real*8 r_e2 ! eccentricity of earth ellipsoid + end type ellipsoid + type(ellipsoid) :: elp + + elp%r_a=6378137.0 + elp%r_e2=0.0066943799901499996 + r_a=6378137.0 + r_e2=0.0066943799901499996 + + r_p = sqrt(r_a**2-r_a**2*r_e2) + pi = 4.d0*atan2(1.d0,1.d0) + + if(iargc().lt.1)then + print *,'Usage: lookvector *.orbtiming' + call exit(1) + end if + if(iargc().ge.1)call getarg(1,posfile) + + ! does the position file exist? + call stat(posfile,buff,status) + if(status.ne.0)then + print *,'Orbtiming file does not exist. ',status + call exit(1) + end if + + ! read in the orbtiming file - usually state vectors at 10 sec centers + open(22,file=posfile,action='read') + read(22,*)startime + read(22,*)endtime + read(22,*)nvect + read(22,*)nvect ! only 4th one counts + allocate (timeorbit(nvect),xx(3,nvect),vv(3,nvect)) + do i=1,nvect + read(22,*)timeorbit(i),xx(:,i),vv(:,i) + ! print *,timeorbit(i),orbitfile + end do + close(22) + timedelta=timeorbit(2)-timeorbit(1) + + ! estimate dem spacing at center of DEM + open(21,file='dem.rsc',action='read') + read(21,'(a)')str + read(str(15:60),*)demwidth + read(21,'(a)')str + read(str(15:60),*)demlength + read(21,'(a)')str + read(str(15:60),*)firstlon + read(21,'(a)')str + read(str(15:60),*)firstlat + read(21,'(a)')str + read(str(15:60),*)deltalon + read(21,'(a)')str + read(str(15:60),*)deltalat + close(21) + + ! print *,'DEM parameters:' + ! print *,demwidth,demlength,firstlon,firstlat,deltalon,deltalat + + ! allocate array to hold look vectors + allocate(lookvec(3,demwidth)) + ! open lookvector file + i=index(posfile,'.orbtiming') + open(41,file=posfile(1:i)//'lookvector',access='stream') + + ! first do some initialization of variables to use in dem loop + ! find the closest state vector to center of scene + llh(1)=(firstlat+(demlength/2)*deltalat)*pi/180. + llh(2)=(firstlon+(demwidth/2)*deltalon)*pi/180. + llh(3)=0. + llhmidpoint=llh + call latlon(elp, xyz0, llh, 1) ! location of dem center, zero height +! call latlon(elp, xyz0,llhtest,2) +! print *,'llh dem midpoint, zero height ',llhtest(1)*180/3.14159,llhtest(2)*180/3.14159,llhtest(3) + ! get local earth radius + ! print *,r_a, r_p, llh(1), cos(llh(1)), sin(llh(1)) + rnum=(r_a**2*cos(llh(1)))**2+(r_p**2*sin(llh(1)))**2 + rden=(r_a*cos(llh(1)))**2+(r_p*sin(llh(1)))**2 + r_local=sqrt(rnum/rden) +! print *,'r local ',r_local + + jclose=0 + rclose=1.e20 + do i=1,nvect + xyzsat=xx(:,i) + if(rclose.gt.sqrt(dot_product(xyzsat-xyz0,xyzsat-xyz0)))then + rclose=sqrt(dot_product(xyzsat-xyz0,xyzsat-xyz0)) + jclose=i + xyzsatmid=xyzsat ! xyzsatmid now contains point of closest approach +! print *,i,jclose,rclose + end if + end do +! print *,'state vector at closest approach ',jclose,rclose + if (jclose.lt.2.or.jclose.gt.nvect-1)then + print *,'precise orbtiming range does not contain proper minimum' + call exit + end if + + q=sqrt(dot_product(xx(:,jclose-1)-xyz0,xx(:,jclose-1)-xyz0)) + qq=sqrt(dot_product(xx(:,jclose+1)-xyz0,xx(:,jclose+1)-xyz0)) + if(q.le.qq)then + jclose=jclose-1 + end if + if (jclose.lt.2)then + print *,'precise orbtimimg range does not contain proper minimum' + call exit + end if +! print *,'state vector at closest approach ',jclose,rclose + xxmid=xx(:,jclose) ! save position and velocity at state vector minimum + vvmid=vv(:,jclose) + timemid=timeorbit(jclose) +! print *,'time xx vv ',timemid,xxmid,vvmid + + ! loop over dem + do kk=1,demlength + lookvec=0. ! zero out lookvec array for each line + + !$OMP PARALLEL DO private(llhtouse,llh,rclose) & + !$OMP private(xyzsat,xyzsatmid,xyz0) & + !$OMP private(time,ilocation,xyzvel,ret,llhtest,rlook,uxyz,xyz1,ue,xspace) & + !$OMP private(uenu,un,yspace,uu,geohdg,r,uproj,cosalpha,cosalphaprime,deltar) & + !$OMP shared(kk,pi,firstlat,firstlon,deltalat,deltalon,elp,nvect) & + !$OMP shared(xx,vv,timeorbit,timedelta,xxmid,vvmid,timemid,r_local) + + do k=1,demwidth + llhtouse(1)=(firstlat+(kk-1)*deltalat)*pi/180. + llhtouse(2)=(firstlon+(k-1)*deltalon)*pi/180. + llhtouse(3)=0. + llh=llhtouse + call latlon(elp, xyz0, llh, 1) ! location of dem point, zero height + ! now iterate to find closest range to this dem point + call orbitrangetime(xyz0,timeorbit,xx,vv,nvect,timemid,xxmid,vvmid,time,rclose) + ilocation=(time-timeorbit(1))/timedelta + call orbithermite(xx(1,ilocation-1),vv(1,ilocation-1),timeorbit(ilocation-1),time,xyzsatmid,xyzvel) + +! print *,'Closest approach range, time: ',rclose,time + ret=unit(xyzvel) +! print *,'unit vector velocity at closest approach: ',xyzvel + + ! sanity check on llh values of satellite, dem +! call latlon(elp, xyzsatmid,llhtest,2) +! print *,'llh satellite location ',llhtest(1)*180/3.14159,llhtest(2)*180/3.14159,llhtest(3) +! call latlon(elp, xyz0,llhtest,2) +! print *,'llh dem midpoint, zero height ',llhtest(1)*180/3.14159,llhtest(2)*180/3.14159,llhtest(3) + +! print *,'xyz coords of satellite closest approach ',xyzsatmid +! print *,'xyz coords of dem midpoint, zero height ',xyz0 + rlook=xyzsatmid-xyz0 +! print *,'Vector surface to satellite, xyz coords ',rlook + uxyz=rlook + ret=unit(uxyz) +! print *,'unit look vector to satellite ',uxyz + + llh=llhtouse + llh(2)=llh(2)+deltalon*pi/180. + call latlon(elp, xyz1, llh, 1) + ! print *,'vector to east ',xyz1-xyz0 + ue=xyz1-xyz0 + ret=unit(ue) +! print *,'unit vector east, xyz coords ',ue + + xspace=r_local*acos(dot_product(xyz0,xyz1)/sqrt(dot_product(xyz0,xyz0)*dot_product(xyz1,xyz1))) +! print *,'xyz1 ',xyz1 +! print *,'xspace ',r_local*acos(dot_product(xyz0,xyz1)/sqrt(dot_product(xyz0,xyz0)*dot_product(xyz1,xyz1))) + + llh=llhtouse + llh(1)=llh(1)-deltalat*pi/180. + call latlon(elp, xyz1, llh, 1) + ! print *,'vector to north ',xyz1-xyz0 + un=xyz1-xyz0 + ret=unit(un) +! print *,'unit vector north, xyz coords ',un + yspace=r_local*acos(dot_product(xyz0,xyz1)/sqrt(dot_product(xyz0,xyz0)*dot_product(xyz1,xyz1))) + + llh=llhtouse + llh(3)=10. + call latlon(elp, xyz1, llh, 1) ! up direction + uu=xyz1-xyz0 + ret=unit(uu) + ! print *,'unit vector up, xyz coords ',uu + + r=sqrt(dot_product(xyzsatmid-xyz0,xyzsatmid-xyz0)) +! print *,'Unit look vector, ground to satellite: ',(xyzsatmid-xyz0)/r + uxyz=xyzsatmid-xyz0 + ret=unit(uxyz) +! print *,'unit look vector xyz coords ',uxyz + + uenu(1)=dot_product(uxyz,ue) + uenu(2)=dot_product(uxyz,un) + uenu(3)=dot_product(uxyz,uu) +! print *,'unit look vector enu coords: ',uenu + ! project unit vector to surface and estimate heading + uproj=uenu; + uproj(3)=0; + ret=unit(uproj); +! print *,'Uenu projected onto surface ',uproj + +!!$ cosalpha=(dot_product(xyzsatmid,xyzsatmid)+r_local**2-r**2)/2/r_local/sqrt(dot_product(xyzsatmid,xyzsatmid)) +!!$ cosalphaprime=(dot_product(xyzsatmid,xyzsatmid)+(r_local+1)**2-r**2)/2/(r_local+1)/sqrt(dot_product(xyzsatmid,xyzsatmid)) +!!$ deltar=r_local*(acos(cosalphaprime)-acos(cosalpha)) + ! print *,'r cosalpha cosalphaprime deltar ',r,cosalpha,cosalphaprime,deltar + + ! save look vector to array +! print *,'writing ',k,kk,uenu,demwidth,demlength + lookvec(1:3,k)=uenu + + if ((k.eq.demwidth/2).and.(kk.eq.demlength/2))then ! save some useful summary params for later use + call velocity_heading(xyzsatmid,xyzvel,geohdg) + ! write parameters for product labeling + i=index(posfile,'.orbtiming') + open(31,file=posfile(1:i)//'labels') + write(31,*)geohdg*180./3.14159,xspace,yspace + write(31,*)-(xyzsatmid-xyz0)/r,r + write(31,*)llhmidpoint*180./3.14159265359 + write(31,*)uenu + write(31,*)' Heading(deg) xpacing yspacing' + write(31,*)'Unit Look vector satellite to ground,range' + write(31,*)'LLH scene midpoint' + write(31,*)'Unit look vector ground to satellite e,n, u coords' + close(31) + end if + + end do + !$OMP END PARALLEL DO + + ! write line out + write(41)lookvec + + end do + + ! close outfile + close(41) + + +end program lookvector + +!************** + +subroutine velocity_heading(xyzsat,xyzvel,geohdg) + + ! approximate satellite heading from spherical earth formula + ! for more precise result use geo_hdg code instead, this is good to a fraction of a degre + + implicit none + + real*8 xyzsat(3),xyzvel(3),geohdg ! args + real*8 llhsat(3),xyz0(3),xyz1(3),llh(3) + real*8 ue(3),un(3),uvenu(3) + real*8 pi + + integer ret,unit + + !c types needed + type :: ellipsoid + real*8 r_a ! semi-major axis + real*8 r_e2 ! eccentricity of earth ellipsoid + end type ellipsoid + type(ellipsoid) :: elp + + elp%r_a=6378137.0 + elp%r_e2=0.0066943799901499996 + + pi=4.d0*atan2(1.d0,1.d0) + + ! llh of satellite at current position + call latlon(elp,xyzsat,llhsat,2) + xyz0=xyzsat + + llh(1)=llhsat(1) + llh(2)=llhsat(2)+1.e-4*pi/180 + llh(3)=llhsat(3) + call latlon(elp, xyz1, llh, 1) + ! print *,'change longitude: ',llhsat,llh + ! print *,'in xyz: ',xyz0,xyz1 + + ! print *,'vector to east ',xyz1-xyz0 + ue=xyz1-xyz0 + ret=unit(ue) + ! print *,'unit vector east, xyz coords ',ue + + llh(1)=llhsat(1)+1.e-4*pi/180 + llh(2)=llhsat(2) + llh(3)=llhsat(3) + call latlon(elp, xyz1, llh, 1) + ! print *,'change latitude: ',llhsat,llh + ! print *,'in xyz: ',xyz0,xyz1 + ! print *,'vector to north ',xyz1-xyz0 + un=xyz1-xyz0 + ret=unit(un) + + uvenu(1)=dot_product(xyzvel,ue) + uvenu(2)=dot_product(xyzvel,un) + uvenu(3)=0. + ret=unit(uvenu) + + !print *,timeorbit(i),atan2(uvenu(1),uvenu(2))*180/3.14159,r_geohdg*180/3.14159,' velocity geo_hdg' + + geohdg=atan2(uvenu(1),uvenu(2)) + return +end subroutine velocity_heading + +integer function unit(u) + real*8 u(3) + u=u/sqrt(u(1)**2+u(2)**2+u(3)**2) +end function unit + diff --git a/sentinel/make_intlist.py b/sentinel/make_intlist.py new file mode 100644 index 0000000..ff5c264 --- /dev/null +++ b/sentinel/make_intlist.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 + +# make_intlist.py - generate intlist for all possible SLC primary images, based on SBAS list and geolist +# create the intlists (all in SBAS list, and if geolist given, individual dates) +# For input into making the all_similarity_masks + +import sys +import string +import os +import math + +if len(sys.argv) < 2: + print ('Usage: make_intlist.py sbas_list ') + sys.exit(1) + +sbaslist=sys.argv[1] +geolist='none' +if len(sys.argv)>2: + geolist=sys.argv[2] + +# first create the intlist based on the entire sbaslist +intlist='intlist' +fintlist=open('intlist','w') + +sbasfiles=[] +fsbas=open(sbaslist,'r') +sbas=fsbas.readlines() +for line in sbas: + words=line.split() + primary=words[0] + secondary=words[1] + # get a short names for primary and secondary files + first=primary.find('20') + primaryname=primary[first:first+8] + first=secondary.find('20') + secondaryname=secondary[first:first+8] + + intfile=primaryname+'_'+secondaryname+'.int' + + # Write interferogram name to intlist + fintlist.write(intfile+'\n') + +fintlist.close() + +if geolist != 'none': + fgeos=open(geolist,'r') + geos=fgeos.readlines() + for geo in geos: + words=geo.split('/') + indate = words[-1][:8] + print(indate) + + fintlist=open('intlist_'+indate,'w') + for line in sbas: + words=line.split() + primary=words[0] + secondary=words[1] + # Get a short name for primary and secondary files + first=primary.find('20') + primaryname=primary[first:first+8] + first=secondary.find('20') + secondaryname=secondary[first:first+8] + + # Check to see if interferogram should be in this intlist + if primaryname != indate: + if secondaryname != indate: + continue + + intfile=primaryname+'_'+secondaryname+'.int' + # Write interferogram name to intlist + fintlist.write(intfile+'\n') + fintlist.close() + + fgeos.close() +fsbas.close() + +sys.exit() + diff --git a/sentinel/make_sbas_lists.py b/sentinel/make_sbas_lists.py new file mode 100644 index 0000000..eb7929f --- /dev/null +++ b/sentinel/make_sbas_lists.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 + +# create a list of sbas pairs for each geolist in listofgeolists (nearest and nearest+maxtb) + +import sys +import subprocess +from datetime import datetime +import os + +if len(sys.argv) < 4: + print('Usage: make_sbas_lists.py listofgeolists geopath max_temporal') + sys.exit(1) + +listofgeolists = sys.argv[1] +geopath = sys.argv[2] +maxtemporal=int(sys.argv[3]) + +maxspatial = 10000 + +# open listofgeolists +fgeo = open(listofgeolists,'r') +geos = fgeo.readlines() +fgeo.close() + +# Make sbas lists +for geolist in geos: + + # nearest neighbor + outname = geolist.replace('geolist','sbas').replace('.txt','_nearest').strip() + command = '$PROC_HOME/sentinel/sbas_list_nearest.py '+geolist.strip()+' '+geopath+' 0 10000 '+outname + print(command) + ret=os.system(command) + + # nearest neighbor + maxtb + outname = outname.replace('nearest',str(maxtemporal)) + command = '$PROC_HOME/sentinel/sbas_list_nearest.py '+geolist.strip()+' '+geopath+' '+str(maxtemporal)+' 10000 '+outname + print(command) + ret=os.system(command) + +print ('#### ---- ALL SBAS LISTS WRITTEN ----- #####') diff --git a/sentinel/orbithermite.o b/sentinel/orbithermite.o new file mode 100644 index 0000000..0d422de Binary files /dev/null and b/sentinel/orbithermite.o differ diff --git a/sentinel/orbitrangetime.o b/sentinel/orbitrangetime.o new file mode 100644 index 0000000..8b5e4ce Binary files /dev/null and b/sentinel/orbitrangetime.o differ diff --git a/sentinel/orbitstatevectors.py b/sentinel/orbitstatevectors.py index 56b5f2b..6775f0b 100755 --- a/sentinel/orbitstatevectors.py +++ b/sentinel/orbitstatevectors.py @@ -18,14 +18,10 @@ def readxmlparam(xmllines, param): istart=str1.find('>')+1 istop=str1.find('<') value=str1[istart:istop] - #print line[i:],'\n' - #print istart, istop, '\n' return value def timeinseconds(timestring): -# dt = datetime.strptime(timestring, 'TAI=%Y-%m-%dT%H:%M:%S.%f') dt = datetime.strptime(timestring, 'UTC=%Y-%m-%dT%H:%M:%S.%f') -# dt = datetime.strptime(timestring, 'UT1=%Y-%m-%dT%H:%M:%S.%f') secs=dt.hour*3600+dt.minute*60+dt.second+dt.microsecond/1000000.0 return secs diff --git a/sentinel/process_zip_list.py b/sentinel/process_zip_list.py index ea7c080..cb5816f 100755 --- a/sentinel/process_zip_list.py +++ b/sentinel/process_zip_list.py @@ -15,26 +15,32 @@ # if len(sys.argv) <2: - print ('Usage: process_zip_list.py ziplist gpu_number ') + print ('Usage: process_zip_list.py ziplist ') sys.exit(1) +# Required Parameters ziplist = sys.argv[1] + +# Default Parameters gpu_n=0 +pol = 'vv' + +# Get Optional Parameters if len(sys.argv) > 2: gpu_n=sys.argv[2] -pol='vv' + if len(sys.argv) > 3: pol=sys.argv[3] -# read in files to be processed by this gpu +# read in files to be processed by this gpu and format zipfiles=[] -fziplist=open(ziplist,'r') -zipfiles=fziplist.readlines() -fziplist.close() +with open(ziplist,'r') as fziplist: + zipfiles=fziplist.readlines() + for i in range(len(zipfiles)): zipfiles[i]=zipfiles[i].strip() -#print ('zipfiles: ',zipfiles) + # assign a gpu os.environ['CUDA_DEVICE_ORDER'] = 'PCI_BUS_ID' @@ -46,31 +52,35 @@ char1=zipfile.find('SSV_') if char1 < 0: char1=zipfile.find('SDV_') - print ('This is a dual pol acquisition') + pol='vh' + print (' This is a dual pol acquisition '+pol) else: - print ('This is a single pol acquisition') + pol='vv' + print (' This is a single pol acquisition '+pol) if char1 < 0: char1=zipfile.find('SSH_') if char1 < 0: char1=zipfile.find('SDH_') - print ('This is a dual pol acquisition') + pol='vh' + print(' This is a dual pol acquisition '+pol) else: - print ('This is a single pol acquisition') - # if horizontal switch polarization (this is a hack, to be improved - if pol == 'vv': pol='hh' + print (' This is a single pol acquisition '+pol) + + ## if horizontal switch polarization (this is a hack, to be improved + #if pol == 'vv': + # pol='hh' char2=zipfile[char1:].find('T') scenedate=zipfile[char1+4:char1+char2] doy=datetime.strptime(scenedate, '%Y%m%d').timetuple().tm_yday year=scenedate[0:4] # day of year and year for scene - print ('doy ',doy,' ',year) + print (' doy ',doy,' ',year) if doy > 1: orbitfilestartdate = datetime.strptime(year+' '+str(doy-1),'%Y %j').strftime('%Y%m%d') else: lastdoy=datetime.strptime(str(int(year)-1)+'1231', '%Y%m%d').timetuple().tm_yday orbitfilestartdate = datetime.strptime(str(int(year)-1)+' '+str(lastdoy),'%Y %j').strftime('%Y%m%d') -# print 'orbit file start date: ',orbitfilestartdate command='grep '+orbitfilestartdate+' preciseorbitfiles' proc = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True) (orbitfilename, err) = proc.communicate() @@ -82,17 +92,14 @@ # and process the scene if you have the precise orbit if len(orbitfilename) < 1: - print ('Missing precise orbit file for ',zipfile) + print (' Skipping: Missing precise orbit file for ',zipfile) command = 'echo skipping file, missing precise orbit' else: command='$PROC_HOME/sentinel/sentinel_scene_multigpu.py '+SAFEname+' '+orbitfilename+' '+pol - print (command) + print (' '+command) ret=os.system(command) -# clean extra position files -#command = 'find . -name \*positionburst\* -delete' -#ret=os.system(command) print ('Loop over ziplist'+gpu_n+' complete.') diff --git a/sentinel/processsub.o b/sentinel/processsub.o new file mode 100644 index 0000000..adf34e5 Binary files /dev/null and b/sentinel/processsub.o differ diff --git a/sentinel/processsubcpu.o b/sentinel/processsubcpu.o new file mode 100644 index 0000000..0c72958 Binary files /dev/null and b/sentinel/processsubcpu.o differ diff --git a/sentinel/ps_sbas_igrams_kp_single.py b/sentinel/ps_sbas_igrams_kp_single.py new file mode 100644 index 0000000..c4f6546 --- /dev/null +++ b/sentinel/ps_sbas_igrams_kp_single.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 + +# ps_sbas_igrams_kp_single - create set of sentinel interferogram subsets +# allow looks +# create the intlist +# don't subset if you don't need to + +import sys +import string +import os +import math + +if len(sys.argv) < 7: + print ('Usage: ps_sbas_igrams_kp_single.py sbas_list dem_rsc_file xstart ystart xsize ysize indate ') + sys.exit(1) + +sbaslist=sys.argv[1] +demrscfile=sys.argv[2] +xstart=sys.argv[3] +ystart=sys.argv[4] +xsize=sys.argv[5] +ysize=sys.argv[6] +indate = sys.argv[7] +xlooks='1' + +if len(sys.argv) > 8: + xlooks=sys.argv[8] + +ylooks=xlooks +if len(sys.argv) > 9: + ylooks=sys.argv[9] +commonflag='n' + +##### ----- GENERATE NEW DEM.RSC FILE FOR MULTILOOKED AND/OR EXTRACTED DEM ----- ##### +command = '$PROC_HOME/util/make_ml_demrsc.py '+demrscfile+' '+xstart+' '+ystart+' '+xsize+' '+ysize+' '+xlooks+' '+ylooks +print('\n '+command) +ret=os.system(command) + +# Get original DEM parameters +with open(demrscfile,'r') as rsc: + words=rsc.readlin() + demwidth=words.split()[1] + words=rsc.readline() + demlength=words.split()[1] + + +# sbaslist +intlist=sbaslist.replace('sbas','int') +fintlist=open(intlist,'w') + +sbasfiles=[] +fsbas=open(sbaslist,'r') +sbas=fsbas.readlines() +for line in sbas: + words=line.split() + primary=words[0] + secondary=words[1] +# get a short names for primary and secondary files + first=primary.find('20') + primaryname=primary[first:first+8] + first=secondary.find('20') + secondaryname=secondary[first:first+8] + + if primaryname != indate: + if secondaryname != indate: + continue + +# print 'primary secondary' +# print primary +# print secondary + + intfile=primaryname+'_'+secondaryname+'.int' + ampfile=primaryname+'_'+secondaryname+'.amp' + ccfile=primaryname+'_'+secondaryname+'.cc' + + # check to see if the file already exists + fintlist.write(intfile+'\n') + if os.path.isfile(intfile): + continue + #fintlist.write(intfile) + #fintlist.write('\n') + + flag=0 + if int(xstart) == 1: + if int(ystart) == 1: + if int(xsize) == int(demwidth): + if int(ysize) == int(demlength): + command='$PROC_HOME/int/crossmul '+primary+' '+secondary+' '+intfile+' '+ampfile+' '+xsize+' '+ysize+' 1.e-6 '+str(xlooks)+' '+str(ylooks) + print (command) + ret=os.system(command) + flag=1 + + if flag == 0: + command='$PROC_HOME/util/subsetmph '+primary+' primarypiece '+demwidth+' '+xstart+' '+ystart+' '+xsize+' '+ysize + print (command) + ret = os.system(command) + command='$PROC_HOME/util/subsetmph '+secondary+' secondarypiece '+demwidth+' '+xstart+' '+ystart+' '+xsize+' '+ysize + print (command) + ret = os.system(command) + command='$PROC_HOME/int/crossmul primarypiece secondarypiece '+intfile+' '+ampfile+' '+xsize+' '+ysize+' 1.e-6 '+str(xlooks)+' '+str(ylooks) + print (command) + ret=os.system(command) + + # correlation file next + ret=os.system('$PROC_HOME/int/makecc '+' '+intfile+' '+ampfile+' '+ccfile+' '+str(int((int(xsize)/int(xlooks))))) + +fintlist.close() + +sys.exit() + diff --git a/sentinel/radar_to_xyz.o b/sentinel/radar_to_xyz.o new file mode 100644 index 0000000..9654e44 Binary files /dev/null and b/sentinel/radar_to_xyz.o differ diff --git a/sentinel/regress_igrams_kp.py b/sentinel/regress_igrams_kp.py new file mode 100644 index 0000000..715f9e7 --- /dev/null +++ b/sentinel/regress_igrams_kp.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 + +# regress_igrams - remove elevation-dependent height from unwrapped igrams + +import sys +ver=sys.version_info + + + + + + + +import os +import sys +import math + +if len(sys.argv) < 4: + print ('Usage: regress_igrams.py unwlist demfile len lines location_file') + sys.exit(1) + +filelist=sys.argv[1] +demfile=sys.argv[2] +width=sys.argv[3] +lines=sys.argv[4] +locationfile=sys.argv[5] + +fsbas=open(filelist,'r') +sbas=fsbas.readlines() +for line in sbas: + unwfile = line.rstrip() + words=line.split('/') + outfile=words[-1].rstrip() + if os.path.isfile(outfile)==False: + command='$PROC_HOME/util/regressheight '+unwfile+' '+outfile+' '+demfile+' '+locationfile+' '+width+' '+lines + print(command) + ret=os.system(command) + +sys.exit() + diff --git a/sentinel/sbas_list_nearest.py b/sentinel/sbas_list_nearest.py new file mode 100644 index 0000000..a77fbb0 --- /dev/null +++ b/sentinel/sbas_list_nearest.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 + +# create a list of sbas pairs +# max temporal = 0 generates minimal list of consecutive acquisitions + +import sys +import subprocess +from datetime import datetime +import os + +if len(sys.argv) < 6: + print('Usage: sbas_list_nearest.py geolist geopath max_temporal max_spatial sbas_out') + sys.exit(1) + +geolist = sys.argv[1] +geopath = sys.argv[2] +maxtemporal=float(sys.argv[3]) +maxspatial=float(sys.argv[4]) +outfile = sys.argv[5] + +# open geolist +fgeo = open(geolist,'r') +geos = fgeo.readlines() +fgeo.close() + +## get a list of the geocoded slc files +#command = 'ls -1 '+geopath+'/*.geo' +#print (command) +#proc = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True) +#(geos, err) = proc.communicate() + +# sort geofiles by date order +#geos=geos.split() +names_times=[] +#print(geos) + +for i in range(0,len(geos)): + #words=str(geos[i],'UTF-8').split('/')[-1] + #wordstring=str(words) + words = geos[i].strip().split('/')[-1] + scenedate=words[:8] + #print(scenedate) + + jd=datetime.strptime(str(scenedate), '%Y%m%d').toordinal()+1721424.5 + names_times.append(str(jd)+' '+geos[i].strip()) + +sortedgeos=sorted(names_times) + +# estimate baseline and create a file for the time-baseline plot +ftb=open(outfile,'w') + +# create lists of dates and filenames, write out geolist +#geolist=open('geolist','w') +jdfile=open('jdlist','w') +jdlist=[] +for i in range(0,len(sortedgeos)): +# geos[i]=sortedgeos[i].split()[1] +# geolist.write(geos[i]+'\n') + + jdlist.append(float(sortedgeos[i].split()[0])) + jdfile.write(str(jdlist[i])+'\n') + +#geolist.close() +jdfile.close() +#print ('Julian day range: ',jdlist[0],jdlist[-1]) + +# call the spatial baseline estimator +print ('Estimating baselines...') + +for i in range(0,len(jdlist)): + for j in range(0,i): + # first do the temporal filter + baseline2=int(abs(jdlist[i]-jdlist[j])) + if (i-j==1) or (baseline2 <= maxtemporal): + # # spatial baseline estimator + # orbtimingi=geopath+geos[i].strip().replace('geo','orbtiming') + # orbtimingj=geopath+geos[j].strip().replace('geo','orbtiming') + # command = '$PROC_HOME/sentinel/geo2rdr/estimatebaseline '+orbtimingi+' '+orbtimingj + # print(command) + # proc = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True) + # (baseline1, err) = proc.communicate() + + # if (i-j==1) or (abs(float(baseline1)) <= maxspatial): + geostri=geopath+geos[i].strip() + geostrj=geopath+geos[j].strip() + # ftb.write(geostrj+' '+geostri+' '+str(baseline2)+' '+str(baseline1,"UTF-8")) + ftb.write(geostrj+' '+geostri+' '+str(baseline2)+' 0\n') + + +#if maxtemporal > 0: + +# for i in range(0,len(jdlist)): +# for j in range(0,i): +# # first do temporal filter +# baseline2=abs(jdlist[i]-jdlist[j]) +# if baseline2 <= maxtemporal: + +# # spatial baseline estimator +# orbtimingi=geos[i].strip().replace('geo','orbtiming') +# orbtimingj=geos[j].strip().replace('geo','orbtiming') +# command = '$PROC_HOME/sentinel/geo2rdr/estimatebaseline '+orbtimingi+' '+orbtimingj +# print(command) +# proc = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True) +# (baseline1, err) = proc.communicate() +# #print (command,' baseline1: ',str(baseline1,"UTF-8")) +# if abs(float(baseline1)) <= maxspatial: +# geostri=geos[i] +# geostrj=geos[j] +# ftb.write(geostrj+' '+geostri+' '+str(baseline2)+' '+str(baseline1,"UTF-8")) + +#else: # zero max temporal means minimal list of pairs +# for i in range(1,len(jdlist)): +# j=i-1 +# orbtimingi=geos[i].strip().replace('geo','orbtiming') +# orbtimingj=geos[j].strip().replace('geo','orbtiming') + +# baseline2=abs(jdlist[i]-jdlist[j]) +# geostri=geos[i] +# geostrj=geos[j] +# ftb.write(geostrj+' '+geostri+' '+str(baseline2)+' 100000000'+'\n') + +print ('sbas list written') +ftb.close() diff --git a/sentinel/sentinel_raw_process b/sentinel/sentinel_raw_process new file mode 100644 index 0000000..e2aad05 Binary files /dev/null and b/sentinel/sentinel_raw_process differ diff --git a/sentinel/sentinel_raw_process.o b/sentinel/sentinel_raw_process.o new file mode 100644 index 0000000..18d3ef2 Binary files /dev/null and b/sentinel/sentinel_raw_process.o differ diff --git a/sentinel/sentinel_raw_process_cpu b/sentinel/sentinel_raw_process_cpu new file mode 100644 index 0000000..c5a3d72 Binary files /dev/null and b/sentinel/sentinel_raw_process_cpu differ diff --git a/sentinel/sentinel_raw_process_cpu.o b/sentinel/sentinel_raw_process_cpu.o new file mode 100644 index 0000000..d2eb0e7 Binary files /dev/null and b/sentinel/sentinel_raw_process_cpu.o differ diff --git a/sentinel/sentinel_scene_cpu.py b/sentinel/sentinel_scene_cpu.py index 443f193..57b38fe 100755 --- a/sentinel/sentinel_scene_cpu.py +++ b/sentinel/sentinel_scene_cpu.py @@ -55,7 +55,7 @@ ret=os.system(command) # process the scene, three swaths -command = '$PROC_HOME/sentinel/sentinel_raw_process_cpu '+basename +command = '$PROC_HOME/sentinel//sentinel_raw_process_cpu '+basename print (command) ret=os.system(command) diff --git a/sentinel/sentinel_scene_multigpu.py b/sentinel/sentinel_scene_multigpu.py index b13e4b9..997f9fc 100755 --- a/sentinel/sentinel_scene_multigpu.py +++ b/sentinel/sentinel_scene_multigpu.py @@ -30,22 +30,24 @@ (datfile, err) = proc.communicate() basename=str(datfile[0:len(datfile)-5],'UTF-8') -#print ('basename: ',basename) -#print ('SAFEname: ',SAFEname) - # read the orbitfile statevectors, store in orbtiming.full command = '$PROC_HOME/sentinel/orbitstatevectors.py '+orbitfile+' '+SAFEname print (command) ret=os.system(command) +# Read the params file to get rsc file +with open('params','r') as fparams: + params = fparams.readlines() +rscfile = params[1].strip() + # initialize the slc file -fe=open('elevation.dem.rsc','r') +fe=open(rscfile,'r') words=fe.readline() demwidth=words.split()[1] words=fe.readline() demlength=words.split()[1] fe.close() -#print demwidth, demlength + command = '$PROC_HOME/sentinel/createslc '+demwidth+' '+demlength+' '+SAFEname+'.geo' print (command) ret=os.system(command) diff --git a/sentinel/sentineltimingsub.o b/sentinel/sentineltimingsub.o new file mode 100644 index 0000000..aecfcc0 Binary files /dev/null and b/sentinel/sentineltimingsub.o differ diff --git a/sentinel/tcnbasis.o b/sentinel/tcnbasis.o new file mode 100644 index 0000000..7ae3aa1 Binary files /dev/null and b/sentinel/tcnbasis.o differ diff --git a/sentinel/unitvec.o b/sentinel/unitvec.o new file mode 100644 index 0000000..568ad65 Binary files /dev/null and b/sentinel/unitvec.o differ diff --git a/sentinel_terminal.py b/sentinel_terminal.py index 4aa24ee..57653b1 100755 --- a/sentinel_terminal.py +++ b/sentinel_terminal.py @@ -17,12 +17,14 @@ def raw_input(): return input() print ( "\n"+"Script to Process Sentinel-1 Stack Using Backprojection"+"\n") -print ( "This script allows you to download Sentinel-1 data from the ASF archive and process it to deformation timeseries. As intermediate products it creates geocoded, phase compensated SLC files. Each SLC file has a .geo suffix, and can be downloaded from your own data area in OpenSARlab or from Google drive. Fully reduced time series products are found in the created mydata/sbas directory.") -print ( "\n"+"Step 1. Prepare your working area"+"\n"+"\n"+"Create a working directory for your analysis, and cd to that directory.") +#print ( "This script allows you to download Sentinel-1 data from the ASF archive and process it to deformation timeseries. As intermediate products it creates geocoded, phase compensated SLC files. Each SLC file has a .geo suffix, and can be downloaded from your own data area in OpenSARlab or from Google drive. Fully reduced time series products are found in the created mydata/sbas directory.") +#print ( "\n"+"Step 1. Prepare your working area"+"\n"+"\n"+"Create a working directory for your analysis, and cd to that directory.") -command = "mkdir mydata\n" -print (command) -ret = os.system(command) +#command = "mkdir mydata\n" +#print (command) +#ret = os.system(command) + +# change directory to subdirectory where you have your scenelist and will download the Sentinel-1 files ret = os.chdir('mydata') print ( "\nStep 2. Download Sentinel raw data to your area\n"+ diff --git a/util/filelen.o b/util/filelen.o new file mode 100644 index 0000000..4f12079 Binary files /dev/null and b/util/filelen.o differ diff --git a/util/io.o b/util/io.o new file mode 100644 index 0000000..2c3a311 Binary files /dev/null and b/util/io.o differ diff --git a/util/lsq.o b/util/lsq.o new file mode 100644 index 0000000..dfed8c7 Binary files /dev/null and b/util/lsq.o differ diff --git a/util/mergeslcs b/util/mergeslcs new file mode 100644 index 0000000..8b96a49 Binary files /dev/null and b/util/mergeslcs differ diff --git a/util/nbymi2 b/util/nbymi2 new file mode 100644 index 0000000..400ec11 Binary files /dev/null and b/util/nbymi2 differ diff --git a/util/nbymi2.o b/util/nbymi2.o new file mode 100644 index 0000000..4c3e1eb Binary files /dev/null and b/util/nbymi2.o differ diff --git a/util/regressheight b/util/regressheight new file mode 100644 index 0000000..7518fa7 Binary files /dev/null and b/util/regressheight differ diff --git a/util/splitziplist.py b/util/splitziplist.py index 3b1fbb1..3d37e48 100755 --- a/util/splitziplist.py +++ b/util/splitziplist.py @@ -6,7 +6,7 @@ import os import string -zipfiles='ziplist' +zipfiles='zipfiles' if len(sys.argv)>1: zipfiles=sys.argv[1] ngpus='1' diff --git a/util/subsetmph b/util/subsetmph new file mode 100644 index 0000000..d1fe06b Binary files /dev/null and b/util/subsetmph differ diff --git a/util/yujiepowlooks b/util/yujiepowlooks new file mode 100644 index 0000000..5e5b14b Binary files /dev/null and b/util/yujiepowlooks differ diff --git a/util/yujiepowlooks.f90 b/util/yujiepowlooks.f90 new file mode 100644 index 0000000..d358258 --- /dev/null +++ b/util/yujiepowlooks.f90 @@ -0,0 +1,78 @@ +!c powlooks - average looks in power + + use omp_lib + + complex*8, allocatable:: a(:,:),b(:),indata(:,:) + character*90 fin,fout + integer*8 filelen,nbytes,nthrds,ID + real*8:: t1,t2 + + !call OMP_SET_NUM_THREADS(2) + !$omp parallel + n=omp_get_num_threads() + !$omp end parallel + print *, 'Number of threads used: ', n + + if(iargc().lt.4)then + write(*,*)'usage: powlooks infile outfile pixac looksac ' + stop + end if + + call getarg(3,fin) + read(fin,*)na + call getarg(4,fin) + read(fin,*)la + if(iargc().ge.5)then + call getarg(5,fin) + read(fin,*)ld + else + ld=la + end if + call getarg(1,fin) + call getarg(2,fout) + + + open(21,file=fin,form='unformatted',access='direct',recl=na*8*ld) + nbytes=filelen(fin) + print *,nbytes,' bytes in file' + nd=nbytes/8/na + print *,'Lines in file: ',nd + print *,'Output file width, pixels: ',na/la + open(22,file=fout,form='unformatted',access='direct',recl=na/la*8) + + + t1=omp_get_wtime() + !$omp parallel do shared(nd,ld,na,la) private(lineout,b,a,jpix,sum1) + do line=1,nd/ld + !c lineout=(line-1)/ld+1 + if(mod(line,1000).eq.1) print *,line + !c allocate the local arrays + allocate (a(na,ld),b(na)) + + b=cmplx(0.,0.) + +!c take looks down + read(21,rec=line,err=99)a + b=sum(cabs(a)**2,2) + +!c take looks across + jpix=0 + do j=1,na,la + jpix=jpix+1 + sum1=0. + do k=0,la-1 + sum1=sum1+real(b(j+k)) + end do + b(jpix)=cmplx(sqrt(sum1),0.) + end do +!c write out data + write(22,rec=line)(b(k),k=1,na/la) + 99 continue + deallocate(a,b) + end do + !$omp end parallel do + t2=omp_get_wtime() + + print*,"Time Taken -->", real(t2-t1) + + end