From df729fa8dc4d0b89c3652ac8752ffdbc8e73fd54 Mon Sep 17 00:00:00 2001 From: HarshitGupta11 <50410275+HarshitGupta11@users.noreply.github.com> Date: Fri, 10 May 2019 22:51:21 +0530 Subject: [PATCH 1/5] Add files via upload --- get_data.py | 28 ++++++++++++++++++++++++++++ my.txt | 2 ++ process_data.py | 15 +++++++++++++++ read_data.py | 38 ++++++++++++++++++++++++++++++++++++++ req_data.py | 30 ++++++++++++++++++++++++++++++ 5 files changed, 113 insertions(+) create mode 100644 get_data.py create mode 100644 my.txt create mode 100644 process_data.py create mode 100644 read_data.py create mode 100644 req_data.py diff --git a/get_data.py b/get_data.py new file mode 100644 index 0000000..aebef8d --- /dev/null +++ b/get_data.py @@ -0,0 +1,28 @@ +import sys +import os +from req_data import get_data_file +from read_data import read_data_genome +from process_data import list_dict_genomes + +arg=sys.argv +arg=arg[1:] + +a=[] +d={} + +feature_name='gene' + +if arg[0]=='-d': + a,d=read_data_genome(arg[1],a,d) +elif arg[0]=='-f': + dir_name=get_data_file(arg[1]) + a,d=read_data_genome(dir_name,a,d) + +assert(len(a)==len(d)) + +ld,ldg=list_dict_genomes(a,d) + +assert(len(ld)==len(ldg)) + +for i in range(len(ld)): + assert(len(ld[i])==len(ldg[i])) diff --git a/my.txt b/my.txt new file mode 100644 index 0000000..7a8fd71 --- /dev/null +++ b/my.txt @@ -0,0 +1,2 @@ +ftp://ftp.ensembl.org/pub/release-96/gtf/lepisosteus_oculatus/Lepisosteus_oculatus.LepOcu1.96.gtf.gz +ftp://ftp.ensembl.org/pub/release-96/gtf/mola_mola/Mola_mola.ASM169857v1.96.gtf.gz diff --git a/process_data.py b/process_data.py new file mode 100644 index 0000000..ee8202d --- /dev/null +++ b/process_data.py @@ -0,0 +1,15 @@ +import pandas +import gc + + +ls=[] +ld=[] +def list_dict_genomes(a,n): + for x in a: + ldg={} + uc=list(x["gene_id"]) + for i in range(len(uc)): + ldg[uc[i]]=i + ls.append(uc) + ld.append(ldg) + return ls,ld diff --git a/read_data.py b/read_data.py new file mode 100644 index 0000000..7d9d8ab --- /dev/null +++ b/read_data.py @@ -0,0 +1,38 @@ +import os +import pandas as pd +import gzip +import sys + +def clear_data(x): + x=x.split() + try: + x=x[1] + except: + c=0 + #print(x) + x=x[1:-1] + return x + +def read_data_genome(dir_name,a,dict_ind_genome): + lf=os.listdir(dir_name) + if len(lf)==0: + print("No files in the data directory!!!!!!") + sys.exit(1) + for x in lf: + + data_gene=pd.read_csv(dir_name+"/"+x,compression='gzip',sep='\t',comment='#',header=None) + #print(data_gene.head) + data_gene=data_gene[data_gene[2]=="gene"] + data_gene=data_gene.sort_values(3) + tmp=data_gene[8].str.split(";",expand=True) + tmp=tmp.iloc[:,:5] + data_gene[["gene_id","gene_version","gene_name","gene_source","gene_biotype"]]=tmp + data_gene=data_gene.drop(8,axis=1) + #print(data_gene[0:10]) + for y in ["gene_version","gene_name","gene_source","gene_biotype","gene_id"]: + data_gene[y]=data_gene[y].apply(clear_data) + print(data_gene[0:10]) + a.append(data_gene) + n=x.split(".")[0] + dict_ind_genome[n]=len(a)-1 + return a,dict_ind_genome diff --git a/req_data.py b/req_data.py new file mode 100644 index 0000000..be4ac0b --- /dev/null +++ b/req_data.py @@ -0,0 +1,30 @@ +import os +import pandas +import sys +import urllib.request as urllib +import pandas as pd + + +lf=[] + +dir_name="data" + +def get_data_file(file): + print(file) + if not os.path.isfile(file): + print("The specified file does not exist!!!") + sys.exit(1) + + with open(file,"r")as f: + lf=f.read().splitlines() + + if os.path.exists("data"): + print("Data Directory Already Exists!!!") + + os.mkdir(dir_name) + for x in lf: + fname=x.split("/")[-1] + path=os.path.join(dir_name,fname) + urllib.urlretrieve(x,path) + + return dir_name From 162ea6434265d1fb92b7a3624d123afe2ed8197a Mon Sep 17 00:00:00 2001 From: HarshitGupta11 <50410275+HarshitGupta11@users.noreply.github.com> Date: Fri, 10 May 2019 23:03:34 +0530 Subject: [PATCH 2/5] Update read_data.py --- read_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/read_data.py b/read_data.py index 7d9d8ab..1e77822 100644 --- a/read_data.py +++ b/read_data.py @@ -31,7 +31,7 @@ def read_data_genome(dir_name,a,dict_ind_genome): #print(data_gene[0:10]) for y in ["gene_version","gene_name","gene_source","gene_biotype","gene_id"]: data_gene[y]=data_gene[y].apply(clear_data) - print(data_gene[0:10]) + #print(data_gene[0:10]) a.append(data_gene) n=x.split(".")[0] dict_ind_genome[n]=len(a)-1 From 606b2a041fa52fdc112452388bb0edf44c8af60d Mon Sep 17 00:00:00 2001 From: HarshitGupta11 <50410275+HarshitGupta11@users.noreply.github.com> Date: Sat, 11 May 2019 23:52:42 +0530 Subject: [PATCH 3/5] Add files via upload --- get_data.py | 56 ++++++++++++++++++++++++++++++++++++---------------- main.py | 27 +++++++++++++++++++++++++ my.txt | 1 + read_data.py | 12 +++++++++++ req_data.py | 20 ++++++++----------- 5 files changed, 87 insertions(+), 29 deletions(-) create mode 100644 main.py diff --git a/get_data.py b/get_data.py index aebef8d..2691d81 100644 --- a/get_data.py +++ b/get_data.py @@ -1,28 +1,50 @@ import sys import os -from req_data import get_data_file -from read_data import read_data_genome +from req_data import get_data_file,download_data +from read_data import read_data_genome,read_data_homology from process_data import list_dict_genomes -arg=sys.argv -arg=arg[1:] +def get_data_genome(arg,dir,a,d,ld,ldg): + if arg[0]=='-d': + if arg[4]=="-r": + c=0 + else: + return a,d,ld,ldg + elif arg[0]=='-f': + get_data_file(arg[1],dir) + elif arg[0]=="-nd": + return ld,ldg,a,d -a=[] -d={} + if arg[4]=="-r": + a,d=read_data_genome(dir,a,d) + assert(len(a)==len(d)) -feature_name='gene' + ld,ldg=list_dict_genomes(a,d) -if arg[0]=='-d': - a,d=read_data_genome(arg[1],a,d) -elif arg[0]=='-f': - dir_name=get_data_file(arg[1]) - a,d=read_data_genome(dir_name,a,d) + assert(len(ld)==len(ldg)) -assert(len(a)==len(d)) + for i in range(len(ld)): + assert(len(ld[i])==len(ldg[i])) -ld,ldg=list_dict_genomes(a,d) + return ld,ldg,a,d -assert(len(ld)==len(ldg)) +def get_data_homology(arg,dir,a_h,d_h): + if arg[2]=="-l": + if not os.path.exists(dir): + os.mkdir(dir) + download_data(arg[3],dir) + elif arg[2]=="-f": + get_data_file(arg[3],dir) + elif arg[2]=="-d": + if arg[4]=="-r": + c=0 + else: + return a_h,d_h + elif arg[2]=="-nd": + return a_h,d_h -for i in range(len(ld)): - assert(len(ld[i])==len(ldg[i])) + if arg[4]=="-r": + a_h,d_h=read_data_homology(dir,a_h,d_h) + assert(len(a_h)==len(d_h)) + + return a_h,d_h diff --git a/main.py b/main.py new file mode 100644 index 0000000..34f26ec --- /dev/null +++ b/main.py @@ -0,0 +1,27 @@ +import sys + +from get_data import get_data_homology,get_data_genome + +arg=sys.argv +arg=arg[1:] + +if len(arg)!=5: + print("No. of arguments more or less. Please check") + sys.exit(1) + +a=[] +d={} +a_h=[] +d_h={} +ld=[] +ldg=[] +dir_g="data" + +ld,ldg,a,d=get_data_genome(arg,dir_g,a,d,ld,ldg) +#print(a[0][0:10],"\n",a[2][0:10],"\n",d,"\n",ld[0][0:10],"\n",ld[2][0:10]) +dir_hom="data_homology" + +a_h,d_h=get_data_homology(arg,dir_hom,a_h,d_h) +#print(a_h[0][0:10],"\n",d_h) + +x=input() diff --git a/my.txt b/my.txt index 7a8fd71..6263a18 100644 --- a/my.txt +++ b/my.txt @@ -1,2 +1,3 @@ ftp://ftp.ensembl.org/pub/release-96/gtf/lepisosteus_oculatus/Lepisosteus_oculatus.LepOcu1.96.gtf.gz ftp://ftp.ensembl.org/pub/release-96/gtf/mola_mola/Mola_mola.ASM169857v1.96.gtf.gz +ftp://ftp.ensembl.org/pub/release-96/gtf/homo_sapiens/Homo_sapiens.GRCh38.96.gtf.gz diff --git a/read_data.py b/read_data.py index 1e77822..cfc1ca7 100644 --- a/read_data.py +++ b/read_data.py @@ -36,3 +36,15 @@ def read_data_genome(dir_name,a,dict_ind_genome): n=x.split(".")[0] dict_ind_genome[n]=len(a)-1 return a,dict_ind_genome + +def read_data_homology(dir,a_h,d_h): + lf=os.listdir(dir) + if len(lf)==0: + print("No Files in the Directory!!!!!!!") + sys.exit(1) + for x in lf: + data=pd.read_csv(dir+"/"+x,compression='gzip',sep='\t') + a_h.append(data) + n=x.split(".")[0] + d_h[n]=len(a_h)-1 + return a_h,d_h diff --git a/req_data.py b/req_data.py index be4ac0b..82016c5 100644 --- a/req_data.py +++ b/req_data.py @@ -7,10 +7,12 @@ import pandas as pd lf=[] -dir_name="data" +def download_data(x,dir_name): + fname=x.split("/")[-1] + path=os.path.join(dir_name,fname) + urllib.urlretrieve(x,path) -def get_data_file(file): - print(file) +def get_data_file(file,dir): if not os.path.isfile(file): print("The specified file does not exist!!!") sys.exit(1) @@ -18,13 +20,7 @@ def get_data_file(file): with open(file,"r")as f: lf=f.read().splitlines() - if os.path.exists("data"): - print("Data Directory Already Exists!!!") - - os.mkdir(dir_name) + if not os.path.exists(dir): + os.mkdir(dir) for x in lf: - fname=x.split("/")[-1] - path=os.path.join(dir_name,fname) - urllib.urlretrieve(x,path) - - return dir_name + download_data(x,dir) From d495c1d1ed08786c29100d49dbeaf43ef8cfcaf0 Mon Sep 17 00:00:00 2001 From: HarshitGupta11 <50410275+HarshitGupta11@users.noreply.github.com> Date: Sun, 12 May 2019 00:15:13 +0530 Subject: [PATCH 4/5] Test Commit --- main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index 34f26ec..59d84bd 100644 --- a/main.py +++ b/main.py @@ -18,7 +18,7 @@ ldg=[] dir_g="data" ld,ldg,a,d=get_data_genome(arg,dir_g,a,d,ld,ldg) -#print(a[0][0:10],"\n",a[2][0:10],"\n",d,"\n",ld[0][0:10],"\n",ld[2][0:10]) +#print(a[0][0:10],"\n",a[2][0:10],"\n",d,"\n",ld[0][0:10],"\n",ld[2][0:10])~~~~~~~ dir_hom="data_homology" a_h,d_h=get_data_homology(arg,dir_hom,a_h,d_h) From 8ffbb5d382fa1c53081662de8d7d87e2b87a23b7 Mon Sep 17 00:00:00 2001 From: HarshitGupta11 <50410275+HarshitGupta11@users.noreply.github.com> Date: Sun, 12 May 2019 00:18:15 +0530 Subject: [PATCH 5/5] Update main.py --- main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index 59d84bd..34f26ec 100644 --- a/main.py +++ b/main.py @@ -18,7 +18,7 @@ ldg=[] dir_g="data" ld,ldg,a,d=get_data_genome(arg,dir_g,a,d,ld,ldg) -#print(a[0][0:10],"\n",a[2][0:10],"\n",d,"\n",ld[0][0:10],"\n",ld[2][0:10])~~~~~~~ +#print(a[0][0:10],"\n",a[2][0:10],"\n",d,"\n",ld[0][0:10],"\n",ld[2][0:10]) dir_hom="data_homology" a_h,d_h=get_data_homology(arg,dir_hom,a_h,d_h)