verena/sw-k-means
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Compare commits

base: verena:21f0a857bec98044513638c123b089b2eab22447
Branches Tags
verena:commented
verena:main
..
compare: verena:a31c1cf0f879c40690471892e470e8acfc0ed348
Branches Tags
verena:commented
verena:main

No commits in common. "21f0a857bec98044513638c123b089b2eab22447" and "a31c1cf0f879c40690471892e470e8acfc0ed348" have entirely different histories.
21f0a857be ... a31c1cf0f8

13 changed files with 33 additions and 93588 deletions
Download patch file Download diff file Expand all files Collapse all files

8795
datadir/ACE_SWICS/ACE_SWICS_Data_2001.txt
View file

File diff suppressed because it is too large Load diff

8785
datadir/ACE_SWICS/ACE_SWICS_Data_2002.txt
View file

File diff suppressed because it is too large Load diff

8702
datadir/ACE_SWICS/ACE_SWICS_Data_2003.txt
View file

File diff suppressed because it is too large Load diff

8780
datadir/ACE_SWICS/ACE_SWICS_Data_2004.txt
View file

File diff suppressed because it is too large Load diff

8801
datadir/ACE_SWICS/ACE_SWICS_Data_2005.txt
View file

File diff suppressed because it is too large Load diff

8804
datadir/ACE_SWICS/ACE_SWICS_Data_2006.txt
View file

File diff suppressed because it is too large Load diff

8799
datadir/ACE_SWICS/ACE_SWICS_Data_2007.txt
View file

File diff suppressed because it is too large Load diff

8821
datadir/ACE_SWICS/ACE_SWICS_Data_2008.txt
View file

File diff suppressed because it is too large Load diff

8805
datadir/ACE_SWICS/ACE_SWICS_Data_2009.txt
View file

File diff suppressed because it is too large Load diff

8801
datadir/ACE_SWICS/ACE_SWICS_Data_2010.txt
View file

File diff suppressed because it is too large Load diff

5634
datadir/ACE_SWICS/ACE_SWICS_Data_2011.txt
View file

File diff suppressed because it is too large Load diff

78
loadData_1h.py
View file

@ -30,10 +30,9 @@ class simpledata(object):
datadict = self.rebin(datadict,keys, safe) datadict = self.rebin(datadict,keys, safe)
safe=(~np.isnan(datadict["dsw"]))&(~np.isnan(datadict["vsw"]))&(~np.isnan(datadict["tsw"]))&(~np.isnan(datadict["B"]))&(~np.isnan(datadict["dO7_6"]))&(~np.isnan(datadict["colage"]))&(~np.isnan(datadict["mcsFe"]))&(~np.isinf(datadict["dsw"]))&(~np.isinf(datadict["vsw"]))&(~np.isinf(datadict["tsw"]))&(~np.isinf(datadict["B"]))&(~np.isinf(datadict["dO7_6"]))&(~np.isinf(datadict["colage"]))&(~np.isinf(datadict["mcsFe"]))&(datadict["colage"]>0)&(datadict["mcsFe"]>0)&(datadict["dO7_6"]>0)&(datadict["totalCountsFe"]>10) safe=(~np.isnan(datadict["dsw"]))&(~np.isnan(datadict["vsw"]))&(~np.isnan(datadict["tsw"]))&(~np.isnan(datadict["B"]))&(~np.isnan(datadict["dO7_6"]))&(~np.isnan(datadict["colage"]))&(~np.isnan(datadict["mcsFe"]))&(~np.isinf(datadict["dsw"]))&(~np.isinf(datadict["vsw"]))&(~np.isinf(datadict["tsw"]))&(~np.isinf(datadict["B"]))&(~np.isinf(datadict["dO7_6"]))&(~np.isinf(datadict["colage"]))&(~np.isinf(datadict["mcsFe"]))&(datadict["colage"]>0)&(datadict["mcsFe"]>0)&(datadict["dO7_6"]>0)&(datadict["totalCountsFe"]>10)
if off_data: if off_data:
safe=(~np.isnan(datadict["dsw"]))&(~np.isnan(datadict["vsw"]))&(~np.isnan(datadict["tsw"]))&(~np.isnan(datadict["B"]))&(~np.isnan(datadict["colage"]))&(~np.isinf(datadict["dsw"]))&(~np.isinf(datadict["vsw"]))&(~np.isinf(datadict["tsw"]))&(~np.isinf(datadict["B"]))&(~np.isinf(datadict["colage"]))&(datadict["colage"]>0) keys=["yeartime", "vsw", "dsw", "tsw", "B", "colage", "dO7_6", "ldO7_6", "elO7_6", "O_error", "mcsFe", "emcsFe","cor_hole", "sec_rev", "stream_belt", "ICME", "totalCountsFe"]
keys=["yeartime", "vsw", "dsw", "tsw", "B", "colage", "cor_hole", "sec_rev", "stream_belt", "ICME" ]
datadict = get_off_data( datadict, safe, keys) datadict = get_off_data( datadict, safe, keys)
safe=(~np.isnan(datadict["dsw"]))&(~np.isnan(datadict["vsw"]))&(~np.isnan(datadict["tsw"]))&(~np.isnan(datadict["B"]))&(~np.isnan(datadict["dO7_6"]))&(~np.isnan(datadict["colage"]))&(~np.isnan(datadict["mcsFe"]))&(~np.isinf(datadict["dsw"]))&(~np.isinf(datadict["vsw"]))&(~np.isinf(datadict["tsw"]))&(~np.isinf(datadict["B"]))&(~np.isinf(datadict["dO7_6"]))&(~np.isinf(datadict["colage"]))&(~np.isinf(datadict["mcsFe"]))&(datadict["colage"]>0)&(datadict["mcsFe"]>0)&(datadict["dO7_6"]>0) safe=(~np.isnan(datadict["dsw"]))&(~np.isnan(datadict["vsw"]))&(~np.isnan(datadict["tsw"]))&(~np.isnan(datadict["B"]))&(~np.isnan(datadict["dO7_6"]))&(~np.isnan(datadict["colage"]))&(~np.isnan(datadict["mcsFe"]))&(~np.isinf(datadict["dsw"]))&(~np.isinf(datadict["vsw"]))&(~np.isinf(datadict["tsw"]))&(~np.isinf(datadict["B"]))&(~np.isinf(datadict["dO7_6"]))&(~np.isinf(datadict["colage"]))&(~np.isinf(datadict["mcsFe"]))&(datadict["colage"]>0)&(datadict["mcsFe"]>0)&(datadict["dO7_6"]>0)&(datadict["totalCountsFe"]>10)
self.X=np.zeros((datadict["vsw"][safe].shape[0], xlength)) self.X=np.zeros((datadict["vsw"][safe].shape[0], xlength))
@ -43,13 +42,9 @@ class simpledata(object):
self.X[:,4]=datadict["vsw"][safe] self.X[:,4]=datadict["vsw"][safe]
self.X[:,5]=datadict["colage"][safe] self.X[:,5]=datadict["colage"][safe]
self.X[:,6]=datadict["mcsFe"][safe] self.X[:,6]=datadict["mcsFe"][safe]
if logO : if logO:
if off_data: self.X[:,3]=datadict["ldO7_6"][safe]
self.X[:,3]=np.log10(datadict["dO7_6"][safe]) self.O_error = datadict["elO7_6"][safe]
self.O_error = datadict["eO7_6"][safe] # does not work this way ...
else:
self.X[:,3]=datadict["ldO7_6"][safe]
self.O_error = datadict["elO7_6"][safe]
else: else:
self.X[:,3]=datadict["dO7_6"][safe] self.X[:,3]=datadict["dO7_6"][safe]
self.O_error = datadict["eO7_6"][safe] self.O_error = datadict["eO7_6"][safe]
@ -111,7 +106,7 @@ class simpledata(object):
def loadData(timeframe=[1,366], years=np.arange(2001,2011,1), relevantkeys=["yeartime", "time", "year", "dsw", "vsw", "tsw", "B", "dO7_6", "elO7_6","ldO7_6", "colage", "mcsFe","emcsFe", "ICME", "cor_hole", "sec_rev", "stream_belt", "totalCountsFe"], prepath="", path= "datadir/", label="2001-2010", rebin= False, off_data= False ): def loadData(timeframe=[1,366], years=np.arange(2001,2011,1), relevantkeys=["yeartime", "time", "year", "dsw", "vsw", "tsw", "B", "dO7_6", "elO7_6","ldO7_6", "colage", "mcsFe","emcsFe", "ICME", "cor_hole", "sec_rev", "stream_belt", "totalCountsFe"], prepath="", path= "datadir/", label="2001-2010", rebin= True, off_data= False ):
""" """
load data from ASCII file load data from ASCII file
The time period is specified as timeframe (start day of year, end day of year) und year. 2001-2010 The time period is specified as timeframe (start day of year, end day of year) und year. 2001-2010
@ -187,7 +182,7 @@ def getBinned(data, keys, timekey="doy", mask=[None], timebins=[None], delta=1
#print ("counter", counter) #print ("counter", counter)
result[timekey]=shorttime[:-1]#-0.5*delta result[timekey]=shorttime[:-1]#-0.5*delta
result["shorttime"]=shorttime[:-1] result["shorttime"]=shorttime[:-1]
#print(len(timebins)) print(len(timebins))
for key in keys: for key in keys:
#safe=np.ones(time.shape, dtype=bool) #safe=np.ones(time.shape, dtype=bool)
#safe*=(np.array(data[key]>minV))*(np.array(data[key]<maxV))*mask #safe*=(np.array(data[key]>minV))*(np.array(data[key]<maxV))*mask
@ -198,21 +193,17 @@ def getBinned(data, keys, timekey="doy", mask=[None], timebins=[None], delta=1
result[timekey + key]=shorttime[:-1]#-0.5*delta result[timekey + key]=shorttime[:-1]#-0.5*delta
y, hx = np.histogram(time[mask], bins=timebins, weights=data[key][mask]) y, hx = np.histogram(time[mask], bins=timebins, weights=data[key][mask])
#print(y) #print(y)
#print(np.sum(counts > 6)) print(np.sum(counts > 6))
result[key]= np.divide(y, counts, out=np.zeros_like(y), where=counts!=0) # y/counts result[key]= np.divide(y, counts, out=np.zeros_like(y), where=counts!=0) # y/counts
#print(len(result[key])) print(len(result[key]))
result[key][counter==0]=np.nan result[key][counter==0]=np.nan
#print(result) #print(result)
return result return result
def get_off_data(data, mask_safe, keys): def get_off_data(data, mask_safe, keys):
start_year = int(data["yeartime"][0])
start_year = int(data["yeartime"][np.isfinite(data["yeartime"])][0]) end_year = int(data["yeartime"][-1])
end_year = int(data["yeartime"][np.isfinite(data["yeartime"])][-1])
#print(data["yeartime"][np.isfinite(data["yeartime"])])
previoustime = data["yeartime"] previoustime = data["yeartime"]
swicsdata = np.empty([]) swicsdata = np.empty([])
for year in np.arange(start_year, end_year+2): for year in np.arange(start_year, end_year+2):
@ -224,40 +215,21 @@ def get_off_data(data, mask_safe, keys):
swicsdata = np.concatenate((swicsdata, data_temp), axis=0) swicsdata = np.concatenate((swicsdata, data_temp), axis=0)
swicstime = swicsdata[:,5] swicstime = swicsdata[:,4]
swicstime = np.append(swicstime, swicstime[-1] +1/365/24 )
# hourmask = np.diff(swicstime) < (1/365/23)
time = data["yeartime"]
delta = np.median(np.diff(swicstime))
#timebins=np.arange(min(swicstime), max(swicstime), delta)
hourmask = np.diff(swicstime) < (1/365/23) hourmask = np.diff(swicstime) < (1/365/23)
print(data)
#delta = np.median(np.diff(swicstime)) delta = np.median(np.diff(swicstime))
print(delta)
counts,shorttime = np.histogram(time,bins=swicstime) time = data["yeartime"]
# result= getBinned(data, keys, timekey="yeartime", mask=hourmask, timebins=swicstime) result= getBinned(data, keys, timekey="yeartime", mask=mask_safe&hourmask, timebins=swicstime)
result= {}
keepdata = {}
for key in keys:
#print(key)
result[key]=shorttime[:-1]
y, hx = np.histogram(time[mask_safe], bins=swicstime, weights=data[key][mask_safe])
result[key]=y/counts
#print(result[key])
keepdata[key] = result[key][hourmask]
print(key, data[key].shape)
keepdata["dO7_6"] = swicsdata[:,7][hourmask] result["dO7_6"] = swicsdata[:,6][hourmask]
keepdata["eO7_6"] = swicsdata[:,8][hourmask] result["eO7_6"] = swicsdata[:,7][hourmask]
keepdata["mcsFe"]= swicsdata[:,10][hourmask] result["mcsFe"]= swicsdata[:,10][hourmask]
keepdata["emcsFe"]= swicsdata[:,11][hourmask] result["yeartime"] = swicstime[hourmask]
print(len(swicstime))
keepdata["time"]= swicsdata[:,4][hourmask] print(len(result["dsw"]))
keepdata["yeartime"] = swicstime[:-1][hourmask] print(len(result["mcsFe"]))
#print("from "+str(previoustime.shape)+" to " + str(keepdata["yeartime"].shape)) #print("from "+str(previoustime.shape)+" to " + str(keepdata["yeartime"].shape))
return keepdata return result

8
reduced_kmeans.py
View file

@ -23,7 +23,7 @@ from bic import compute_bic
import time import time
NoneType = type(None) NoneType = type(None)
from loadData_1h import loadData, loadDataRed from loadData import loadData, loadDataRed
# update the matplotlib standard parameters # update the matplotlib standard parameters
backend_bases.register_backend('pdf', FigureCanvasPgf) backend_bases.register_backend('pdf', FigureCanvasPgf)
@ -2663,7 +2663,7 @@ if __name__=="__main__":
add_label = '_d' add_label = '_d'
else: else:
ntrials = 100 ntrials = 100
add_label = '_1h_off_SWICS' add_label = ''
if int(option) ==0: if int(option) ==0:
#This option creates all needed directories #This option creates all needed directories
@ -2682,7 +2682,7 @@ if __name__=="__main__":
""" """
if option - int(option) == 0: if option - int(option) == 0:
option = int(option) option = int(option)
data = loadData(off_data = True) data = loadData()
bigExp_general(data, option,8, ntrials = ntrials) bigExp_general(data, option,8, ntrials = ntrials)
else: else:
@ -2721,7 +2721,7 @@ if __name__=="__main__":
elif int(option) == 33: elif int(option) == 33:
if option - int(option) == 0: if option - int(option) == 0:
data = loadData() data = loadData()
#add_label = '_new' add_label = '_new'
else: else:
data = loadData() data = loadData()
data.downsample(samples = samples) data.downsample(samples = samples)