mirror of
https://github.com/Priyatham-sai-chand/compara-deep-learning.git
synced 2026-08-15 04:01:14 -07:00
Add files via upload
This commit is contained in:
HarshitGupta11
GitHub
parent
29bd784782
commit
a58c842610
16 changed files with 971 additions and 821 deletions
|
|
@ -3,152 +3,176 @@ import requests
|
|||
import progressbar
|
||||
import sys
|
||||
|
||||
def update_protein(gene_seq,gene):
|
||||
t=0
|
||||
while(t!=2):
|
||||
|
||||
def update_protein(gene_seq, gene):
|
||||
t = 0
|
||||
while(t != 2):
|
||||
try:
|
||||
server = "https://rest.ensembl.org"
|
||||
ext = "/sequence/id/"+str(gene)+"?type=protein;multiple_sequences=1"
|
||||
ext = "/sequence/id/" + \
|
||||
str(gene) + "?type=protein;multiple_sequences=1"
|
||||
|
||||
r = requests.get(server+ext, headers={ "Content-Type" : "application/json"})
|
||||
r = requests.get(
|
||||
server + ext,
|
||||
headers={
|
||||
"Content-Type": "application/json"})
|
||||
|
||||
if not r.ok:
|
||||
r.raise_for_status()
|
||||
sys.exit()
|
||||
r=r.json()
|
||||
if len(r)==1:
|
||||
r=dict(r[0])
|
||||
gene_seq[gene]=str(r["seq"])
|
||||
r = r.json()
|
||||
if len(r) == 1:
|
||||
r = dict(r[0])
|
||||
gene_seq[gene] = str(r["seq"])
|
||||
return
|
||||
else:
|
||||
maxi=0
|
||||
maxlen=0
|
||||
maxi = 0
|
||||
maxlen = 0
|
||||
for i in range(len(r)):
|
||||
m=r[i]
|
||||
m=dict(m)
|
||||
if len(m["seq"])>maxlen:
|
||||
maxi=i
|
||||
r=dict(r[maxi])
|
||||
gene_seq[gene]=str(r["seq"])
|
||||
m = r[i]
|
||||
m = dict(m)
|
||||
if len(m["seq"]) > maxlen:
|
||||
maxi = i
|
||||
r = dict(r[maxi])
|
||||
gene_seq[gene] = str(r["seq"])
|
||||
return
|
||||
except :
|
||||
t+=1
|
||||
#print("\nError:",e)
|
||||
except BaseException:
|
||||
t += 1
|
||||
# print("\nError:",e)
|
||||
continue
|
||||
gene_seq[gene]=""
|
||||
gene_seq[gene] = ""
|
||||
|
||||
|
||||
def update_rest_protein(data):
|
||||
gids={}
|
||||
with open("processed/not_found.json","r") as file:
|
||||
gids=dict(json.load(file))
|
||||
gids = {}
|
||||
with open("processed/not_found.json", "r") as file:
|
||||
gids = dict(json.load(file))
|
||||
|
||||
gids=list(gids.keys())
|
||||
gids = list(gids.keys())
|
||||
|
||||
geneseq={}
|
||||
geneseq = {}
|
||||
|
||||
server = "https://rest.ensembl.org"
|
||||
ext = "/sequence/id?type=protein"
|
||||
headers={ "Content-Type" : "application/json", "Accept" : "application/json"}
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
"Accept": "application/json"}
|
||||
|
||||
for i in progressbar.progressbar(range(0,len(gids)-50,50)):
|
||||
ids=dict(ids=list(gids[i:i+50]))
|
||||
for i in progressbar.progressbar(range(0, len(gids) - 50, 50)):
|
||||
ids = dict(ids=list(gids[i:i + 50]))
|
||||
while(1):
|
||||
try:
|
||||
r = requests.post(server+ext, headers=headers, data=str(json.dumps(ids)))
|
||||
r = requests.post(
|
||||
server + ext,
|
||||
headers=headers,
|
||||
data=str(
|
||||
json.dumps(ids)))
|
||||
if not r.ok:
|
||||
r.raise_for_status()
|
||||
gs=r.json()
|
||||
tgs={}
|
||||
gs = r.json()
|
||||
tgs = {}
|
||||
for g in gs:
|
||||
tgs[g["query"]]=g["seq"]
|
||||
tgs[g["query"]] = g["seq"]
|
||||
geneseq.update(tgs)
|
||||
break
|
||||
except Exception as e:
|
||||
print("Error:",e)
|
||||
print("Error:", e)
|
||||
continue
|
||||
|
||||
data.update(geneseq)
|
||||
for genes in gids:
|
||||
try:
|
||||
_=data[genes]
|
||||
except:
|
||||
_ = data[genes]
|
||||
except BaseException:
|
||||
print(genes)
|
||||
update_protein(data,genes)
|
||||
update_protein(data, genes)
|
||||
|
||||
print("Gene Sequences Updated Successfully")
|
||||
return data
|
||||
|
||||
def update(gene_seq,gene):
|
||||
t=0
|
||||
while(t!=2):
|
||||
|
||||
|
||||
def update(gene_seq, gene):
|
||||
t = 0
|
||||
while(t != 2):
|
||||
try:
|
||||
server = "https://rest.ensembl.org"
|
||||
ext = "/sequence/id/"+str(gene)+"?type=cds;multiple_sequences=1"
|
||||
ext = "/sequence/id/" + \
|
||||
str(gene) + "?type=cds;multiple_sequences=1"
|
||||
|
||||
r = requests.get(server+ext, headers={ "Content-Type" : "application/json"})
|
||||
r = requests.get(
|
||||
server + ext,
|
||||
headers={
|
||||
"Content-Type": "application/json"})
|
||||
|
||||
if not r.ok:
|
||||
r.raise_for_status()
|
||||
sys.exit()
|
||||
r=r.json()
|
||||
if len(r)==1:
|
||||
r=dict(r[0])
|
||||
gene_seq[gene]=str(r["seq"])
|
||||
r = r.json()
|
||||
if len(r) == 1:
|
||||
r = dict(r[0])
|
||||
gene_seq[gene] = str(r["seq"])
|
||||
return
|
||||
else:
|
||||
maxi=0
|
||||
maxlen=0
|
||||
maxi = 0
|
||||
maxlen = 0
|
||||
for i in range(len(r)):
|
||||
m=r[i]
|
||||
m=dict(m)
|
||||
if len(m["seq"])>maxlen:
|
||||
maxi=i
|
||||
r=dict(r[maxi])
|
||||
gene_seq[gene]=str(r["seq"])
|
||||
m = r[i]
|
||||
m = dict(m)
|
||||
if len(m["seq"]) > maxlen:
|
||||
maxi = i
|
||||
r = dict(r[maxi])
|
||||
gene_seq[gene] = str(r["seq"])
|
||||
return
|
||||
except Exception as e:
|
||||
t+=1
|
||||
#print("\nError:",e)
|
||||
except BaseException:
|
||||
t += 1
|
||||
# print("\nError:",e)
|
||||
continue
|
||||
gene_seq[gene]=""
|
||||
gene_seq[gene] = ""
|
||||
|
||||
def update_rest(data,fname):
|
||||
gids={}
|
||||
with open("processed/not_found_"+fname+".json","r") as file:
|
||||
gids=dict(json.load(file))
|
||||
|
||||
gids=list(gids.keys())
|
||||
def update_rest(data, fname):
|
||||
gids = {}
|
||||
with open("processed/not_found_" + fname + ".json", "r") as file:
|
||||
gids = dict(json.load(file))
|
||||
|
||||
geneseq={}
|
||||
gids = list(gids.keys())
|
||||
|
||||
geneseq = {}
|
||||
|
||||
server = "https://rest.ensembl.org"
|
||||
ext = "/sequence/id?type=cds"
|
||||
headers={ "Content-Type" : "application/json", "Accept" : "application/json"}
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
"Accept": "application/json"}
|
||||
|
||||
for i in progressbar.progressbar(range(0,len(gids)-50,50)):
|
||||
ids=dict(ids=list(gids[i:i+50]))
|
||||
for i in progressbar.progressbar(range(0, len(gids) - 50, 50)):
|
||||
ids = dict(ids=list(gids[i:i + 50]))
|
||||
while(1):
|
||||
try:
|
||||
r = requests.post(server+ext, headers=headers, data=str(json.dumps(ids)))
|
||||
r = requests.post(
|
||||
server + ext,
|
||||
headers=headers,
|
||||
data=str(
|
||||
json.dumps(ids)))
|
||||
if not r.ok:
|
||||
r.raise_for_status()
|
||||
gs=r.json()
|
||||
tgs={}
|
||||
gs = r.json()
|
||||
tgs = {}
|
||||
for g in gs:
|
||||
tgs[g["query"]]=g["seq"]
|
||||
tgs[g["query"]] = g["seq"]
|
||||
geneseq.update(tgs)
|
||||
break
|
||||
except Exception as e:
|
||||
print("Error:",e)
|
||||
print("Error:", e)
|
||||
continue
|
||||
|
||||
data.update(geneseq)
|
||||
for genes in gids:
|
||||
try:
|
||||
_=data[genes]
|
||||
except:
|
||||
_ = data[genes]
|
||||
except BaseException:
|
||||
print(genes)
|
||||
update(data,genes)
|
||||
update(data, genes)
|
||||
|
||||
print("Gene Sequences Updated Successfully")
|
||||
return data
|
||||
return data
|
||||
|
|
|
|||
|
|
@ -1,15 +1,12 @@
|
|||
import pandas as pd
|
||||
import requests
|
||||
import sys
|
||||
import pickle
|
||||
from get_data import get_data_genome
|
||||
|
||||
dir_g="data"
|
||||
cmap,cimap,ld,ldg,a,d=get_data_genome(dir_g)
|
||||
dir_g = "data"
|
||||
cmap, cimap, ld, ldg, a, d = get_data_genome(dir_g)
|
||||
|
||||
data=dict(cmap=cmap,cimap=cimap,ld=ld,ldg=ldg,a=a,d=d)
|
||||
data = dict(cmap=cmap, cimap=cimap, ld=ld, ldg=ldg, a=a, d=d)
|
||||
|
||||
with open("genome_maps","wb") as file:
|
||||
pickle.dump(data,file)
|
||||
with open("genome_maps", "wb") as file:
|
||||
pickle.dump(data, file)
|
||||
|
||||
print("Genome Maps Created Successfully.")
|
||||
print("Genome Maps Created Successfully.")
|
||||
|
|
|
|||
|
|
@ -1,7 +1,4 @@
|
|||
import pandas as pd
|
||||
import numpy as np
|
||||
import json
|
||||
import gc
|
||||
import numpy as np
|
||||
import pickle
|
||||
import os
|
||||
import sys
|
||||
|
|
@ -9,88 +6,101 @@ from tree_data import create_tree_data
|
|||
from process_negative import read_database_txt
|
||||
from select_data import read_db_homology
|
||||
|
||||
def read_data_homology(dirname,nfname):
|
||||
lf=os.listdir(dirname)
|
||||
if len(lf)==0:
|
||||
|
||||
def read_data_homology(dirname, nfname):
|
||||
lf = os.listdir(dirname)
|
||||
if len(lf) == 0:
|
||||
print("No Files in the Directory!!!!!!!")
|
||||
sys.exit(1)
|
||||
a_h=[]
|
||||
d_h=[]
|
||||
a_h = []
|
||||
d_h = []
|
||||
for x in lf:
|
||||
df,n=read_db_homology(dirname,x)
|
||||
n=n.split()[0]
|
||||
df, n = read_db_homology(dirname, x)
|
||||
n = n.split()[0]
|
||||
try:
|
||||
indexes=np.load("processed/synteny_matrices/"+n+"_indexes.npy")
|
||||
except:
|
||||
print("Incomplete data for:",n)
|
||||
df=df.loc[indexes]
|
||||
indexes = np.load(
|
||||
"processed/synteny_matrices/" +
|
||||
n +
|
||||
"_indexes.npy")
|
||||
except BaseException:
|
||||
print("Incomplete data for:", n)
|
||||
df = df.loc[indexes]
|
||||
a_h.append(df)
|
||||
d_h.append(n)
|
||||
#read the negative dataset
|
||||
df=read_database_txt(nfname)
|
||||
indexes=np.load("processed/synteny_matrices/"+nfname.split(".")[0]+"_indexes.npy")
|
||||
df=df.loc[indexes]
|
||||
# read the negative dataset
|
||||
df = read_database_txt(nfname)
|
||||
indexes = np.load(
|
||||
"processed/synteny_matrices/" +
|
||||
nfname.split(".")[0] +
|
||||
"_indexes.npy")
|
||||
df = df.loc[indexes]
|
||||
a_h.append(df)
|
||||
d_h.append(nfname.split(".")[0])
|
||||
return a_h,d_h
|
||||
return a_h, d_h
|
||||
|
||||
def prepare_features(a_h,d_h,sptree,label):
|
||||
rows=[]
|
||||
smg_name="_synteny_matrices_global.npy"
|
||||
sml_name="_synteny_matrices_local.npy"
|
||||
smi_name="_indexes.npy"
|
||||
dir_name="processed/synteny_matrices/"
|
||||
|
||||
def prepare_features(a_h, d_h, sptree, label):
|
||||
rows = []
|
||||
smg_name = "_synteny_matrices_global.npy"
|
||||
sml_name = "_synteny_matrices_local.npy"
|
||||
smi_name = "_indexes.npy"
|
||||
dir_name = "processed/synteny_matrices/"
|
||||
for i in range(len(a_h)):
|
||||
df=a_h[i]
|
||||
n=d_h[i]
|
||||
df = a_h[i]
|
||||
n = d_h[i]
|
||||
try:
|
||||
smg=np.load(dir_name+n+smg_name)
|
||||
sml=np.load(dir_name+n+sml_name)
|
||||
indexes=np.load(dir_name+n+smi_name)
|
||||
except:
|
||||
print("Incomplete data for:",n)
|
||||
smg = np.load(dir_name + n + smg_name)
|
||||
sml = np.load(dir_name + n + sml_name)
|
||||
indexes = np.load(dir_name + n + smi_name)
|
||||
except BaseException:
|
||||
print("Incomplete data for:", n)
|
||||
continue
|
||||
df=df.loc[indexes]
|
||||
df = df.loc[indexes]
|
||||
|
||||
branch_length_species,branch_length_homology_species,distance,dist_p_s,dist_p_hs=create_tree_data(sptree,df)
|
||||
assert(len(branch_length_species)==len(df))
|
||||
assert(len(sml)==len(distance))
|
||||
branch_length_species, \
|
||||
branch_length_homology_species, \
|
||||
distance, dist_p_s, dist_p_hs = create_tree_data(
|
||||
sptree, df)
|
||||
assert(len(branch_length_species) == len(df))
|
||||
assert(len(sml) == len(distance))
|
||||
|
||||
for i in range(len(df)):
|
||||
index=indexes[i]
|
||||
row=df.loc[index]
|
||||
r={}
|
||||
r["species"]=row["species"]
|
||||
r["homology_species"]=row["homology_species"]
|
||||
r["gene_stable_id"]=row["gene_stable_id"]
|
||||
r["homology_gene_stable_id"]=row["homology_gene_stable_id"]
|
||||
r["label"]=label[row["homology_type"]]
|
||||
r["global_alignment_matrix"]=smg[i]
|
||||
r["local_alignment_matrix"]=sml[i]
|
||||
r["index_homology_dataset"]=index
|
||||
r["bls"]=branch_length_species[i]
|
||||
r["blhs"]=branch_length_homology_species[i]
|
||||
r["dis"]=distance[i]
|
||||
r["dps"]=dist_p_s[i]
|
||||
r["dphs"]=dist_p_hs[i]
|
||||
index = indexes[i]
|
||||
row = df.loc[index]
|
||||
r = {}
|
||||
r["species"] = row["species"]
|
||||
r["homology_species"] = row["homology_species"]
|
||||
r["gene_stable_id"] = row["gene_stable_id"]
|
||||
r["homology_gene_stable_id"] = row["homology_gene_stable_id"]
|
||||
r["label"] = label[row["homology_type"]]
|
||||
r["global_alignment_matrix"] = smg[i]
|
||||
r["local_alignment_matrix"] = sml[i]
|
||||
r["index_homology_dataset"] = index
|
||||
r["bls"] = branch_length_species[i]
|
||||
r["blhs"] = branch_length_homology_species[i]
|
||||
r["dis"] = distance[i]
|
||||
r["dps"] = dist_p_s[i]
|
||||
r["dphs"] = dist_p_hs[i]
|
||||
rows.append(r)
|
||||
return rows
|
||||
|
||||
|
||||
def main():
|
||||
arg=sys.argv
|
||||
nfname=arg[-1]
|
||||
a_h,d_h=read_data_homology("data_homology",nfname)
|
||||
labels=dict(ortholog_one2one=1,
|
||||
other_paralog=0,
|
||||
non_homolog=2,
|
||||
ortholog_one2many=1,
|
||||
ortholog_many2many=1,
|
||||
within_species_paralog=0,
|
||||
gene_split=4)
|
||||
rows=prepare_features(a_h,d_h,"species_tree.tree",labels)
|
||||
with open("dataset","wb") as file:
|
||||
pickle.dump(rows,file)
|
||||
arg = sys.argv
|
||||
nfname = arg[-1]
|
||||
a_h, d_h = read_data_homology("data_homology", nfname)
|
||||
labels = dict(ortholog_one2one=1,
|
||||
other_paralog=0,
|
||||
non_homolog=2,
|
||||
ortholog_one2many=1,
|
||||
ortholog_many2many=1,
|
||||
within_species_paralog=0,
|
||||
gene_split=4)
|
||||
rows = prepare_features(a_h, d_h, "species_tree.tree", labels)
|
||||
with open("dataset", "wb") as file:
|
||||
pickle.dump(rows, file)
|
||||
print("Dataset_Finalized")
|
||||
|
||||
if __name__=="__main__":
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
|
|||
36
get_data.py
36
get_data.py
|
|
@ -1,26 +1,18 @@
|
|||
import sys
|
||||
import os
|
||||
from read_data import read_data_genome,read_data_homology
|
||||
from process_data import list_dict_genomes,create_chromosome_maps
|
||||
from read_data import read_data_genome
|
||||
from process_data import list_dict_genomes, create_chromosome_maps
|
||||
|
||||
|
||||
def get_data_genome(dir):
|
||||
a=[]
|
||||
d={}
|
||||
ld=[]
|
||||
ldg=[]
|
||||
a,d=read_data_genome(dir,a,d)
|
||||
assert(len(a)==len(d))
|
||||
a = []
|
||||
d = {}
|
||||
ld = []
|
||||
ldg = []
|
||||
a, d = read_data_genome(dir, a, d)
|
||||
assert(len(a) == len(d))
|
||||
print("Creating Maps:")
|
||||
ld,ldg=list_dict_genomes(a,d)
|
||||
cmap,cimap=create_chromosome_maps(a,d)
|
||||
assert(len(ld)==len(ldg))
|
||||
ld, ldg = list_dict_genomes(a, d)
|
||||
cmap, cimap = create_chromosome_maps(a, d)
|
||||
assert(len(ld) == len(ldg))
|
||||
for i in range(len(ld)):
|
||||
assert(len(ld[i])==len(ldg[i]))
|
||||
return cmap,cimap,ld,ldg,a,d
|
||||
|
||||
def get_data_homology(dir):
|
||||
a_h=[]
|
||||
d_h={}
|
||||
a_h,d_h=read_data_homology(dir)
|
||||
assert(len(a_h)==len(d_h))
|
||||
return a_h,d_h
|
||||
assert(len(ld[i]) == len(ldg[i]))
|
||||
return cmap, cimap, ld, ldg, a, d
|
||||
|
|
|
|||
|
|
@ -1,53 +1,53 @@
|
|||
import sys
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import json
|
||||
import os
|
||||
import gc
|
||||
import pickle
|
||||
from select_data import read_db_homology
|
||||
from process_data import create_data_homology_ls
|
||||
|
||||
|
||||
def read_genome_maps():
|
||||
data={}
|
||||
with open("genome_maps","rb") as file:
|
||||
data=pickle.load(file)
|
||||
cmap=data["cmap"]
|
||||
cimap=data["cimap"]
|
||||
ld=data["ld"]
|
||||
ldg=data["ldg"]
|
||||
a=data["a"]
|
||||
d=data["d"]
|
||||
return a,d,ld,ldg,cmap,cimap
|
||||
data = {}
|
||||
with open("genome_maps", "rb") as file:
|
||||
data = pickle.load(file)
|
||||
cmap = data["cmap"]
|
||||
cimap = data["cimap"]
|
||||
ld = data["ld"]
|
||||
ldg = data["ldg"]
|
||||
a = data["a"]
|
||||
d = data["d"]
|
||||
return a, d, ld, ldg, cmap, cimap
|
||||
|
||||
|
||||
def read_data_homology(dirname):
|
||||
lf=os.listdir(dirname)
|
||||
if len(lf)==0:
|
||||
lf = os.listdir(dirname)
|
||||
if len(lf) == 0:
|
||||
print("No Files in the Directory!!!!!!!")
|
||||
sys.exit(1)
|
||||
a_h=[]
|
||||
d_h=[]
|
||||
a_h = []
|
||||
d_h = []
|
||||
for x in lf:
|
||||
df,n=read_db_homology(dirname,x)
|
||||
n=n.split()[0]
|
||||
df, n = read_db_homology(dirname, x)
|
||||
n = n.split()[0]
|
||||
try:
|
||||
indexes=np.load("processed/"+n+"_selected_indexes.npy")
|
||||
except:
|
||||
print("Incomplete data for:",n)
|
||||
df=df.loc[indexes]
|
||||
print(len(df))
|
||||
indexes = np.load("processed/" + n + "_selected_indexes.npy")
|
||||
except BaseException:
|
||||
print("Incomplete data for:", n)
|
||||
df = df.loc[indexes]
|
||||
a_h.append(df)
|
||||
d_h.append(n)
|
||||
return a_h,d_h
|
||||
return a_h, d_h
|
||||
|
||||
|
||||
def main():
|
||||
a,d,ld,ldg,cmap,cimap=read_genome_maps()
|
||||
a, d, ld, ldg, cmap, cimap = read_genome_maps()
|
||||
print("Genome Maps Loaded.")
|
||||
a_h,d_h=read_data_homology("data_homology")
|
||||
a_h, d_h = read_data_homology("data_homology")
|
||||
print("Data Read.")
|
||||
n=3
|
||||
_=create_data_homology_ls(a_h,d_h,n,a,d,ld,ldg,cmap,cimap,1)
|
||||
n = 3
|
||||
_ = create_data_homology_ls(a_h, d_h, n, a, d, ld, ldg, cmap, cimap, 1)
|
||||
print("Neighbor Genes Found and Saved Successfully:)")
|
||||
|
||||
if __name__=="__main__":
|
||||
main()
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
|
|||
226
prediction.py
226
prediction.py
|
|
@ -1,8 +1,5 @@
|
|||
import json
|
||||
import gc
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
import pickle
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
import tensorflow as tf
|
||||
import sys
|
||||
import progressbar
|
||||
|
|
@ -15,101 +12,126 @@ from prepare_synteny_matrix import read_data_synteny
|
|||
from tree_data import create_tree_data
|
||||
from process_data import create_map_list
|
||||
|
||||
|
||||
def read_database(fname):
|
||||
df=pd.read_csv(fname,sep="\t",header=None)
|
||||
label_dict=dict(ortholog_one2one=1,
|
||||
other_paralog=0,
|
||||
non_homolog=2,
|
||||
ortholog_one2many=1,
|
||||
ortholog_many2many=1,
|
||||
within_species_paralog=0,
|
||||
gene_split=4)
|
||||
label=[]
|
||||
for _,row in df.iterrows():
|
||||
df = pd.read_csv(fname, sep="\t", header=None)
|
||||
label_dict = dict(ortholog_one2one=1,
|
||||
other_paralog=0,
|
||||
non_homolog=2,
|
||||
ortholog_one2many=1,
|
||||
ortholog_many2many=1,
|
||||
within_species_paralog=0,
|
||||
gene_split=4)
|
||||
label = []
|
||||
for _, row in df.iterrows():
|
||||
label.append(label_dict[row[7]])
|
||||
df=df.assign(label=label)
|
||||
df=df.drop(7,axis=1)
|
||||
df=df.drop(0,axis=1)
|
||||
df.columns=["gene_stable_id","species","homology_gene_stable_id","homology_species","goc","wga","label"]
|
||||
df = df.assign(label=label)
|
||||
df = df.drop(7, axis=1)
|
||||
df = df.drop(0, axis=1)
|
||||
df.columns = [
|
||||
"gene_stable_id",
|
||||
"species",
|
||||
"homology_gene_stable_id",
|
||||
"homology_species",
|
||||
"goc",
|
||||
"wga",
|
||||
"label"]
|
||||
return df
|
||||
|
||||
def select_data_by_length(df,st,end):
|
||||
|
||||
def select_data_by_length(df, st, end):
|
||||
try:
|
||||
if end<len(df):
|
||||
if st<end:
|
||||
df=df.loc[df.index.values[st:end]]
|
||||
if end < len(df):
|
||||
if st < end:
|
||||
df = df.loc[df.index.values[st:end]]
|
||||
else:
|
||||
raise ValueError()
|
||||
except:
|
||||
except BaseException:
|
||||
print("Making Predictions for the complete dataframe:)")
|
||||
print(len(df))
|
||||
return df
|
||||
|
||||
def create_synteny_features(a_h,d_h,n,a,d,ld,ldg,cmap,cimap,name):
|
||||
lsy=create_data_homology_ls(a_h,d_h,n,a,d,ld,ldg,cmap,cimap,0)
|
||||
gene_sequences=read_gene_sequences(a_h,lsy,"geneseq","prediction_"+name)
|
||||
gene_sequences=update_rest(gene_sequences,"prediction_"+name)
|
||||
|
||||
def create_synteny_features(a_h, d_h, n, a, d, ld, ldg, cmap, cimap, name):
|
||||
lsy = create_data_homology_ls(a_h, d_h, n, a, d, ld, ldg, cmap, cimap, 0)
|
||||
gene_sequences = read_gene_sequences(
|
||||
a_h, lsy, "geneseq", "prediction_" + name)
|
||||
gene_sequences = update_rest(gene_sequences, "prediction_" + name)
|
||||
print("Gene Sequences Loaded.")
|
||||
return lsy,gene_sequences
|
||||
return lsy, gene_sequences
|
||||
|
||||
def threadmaker(nop,df,lsy,gene_sequences,n,name):
|
||||
part=len(df)//nop
|
||||
pr=Procerssrunner()
|
||||
pr.start_processes(nop,df,gene_sequences,lsy,part,n,name)
|
||||
smg,sml,indexes=read_data_synteny(nop,name)
|
||||
sml=np.array(sml)
|
||||
smg=np.array(smg)
|
||||
indexes=np.array(indexes)
|
||||
return sml,smg,indexes
|
||||
|
||||
def get_prediction(smg,sml,indexes,bls,blhs,dis,dps,dphs,model_name):
|
||||
preds=np.zeros((len(smg),3))
|
||||
w=[0.86,0.8,0.06]
|
||||
for i in range(1,4):
|
||||
def threadmaker(nop, df, lsy, gene_sequences, n, name):
|
||||
part = len(df) // nop
|
||||
pr = Procerssrunner()
|
||||
pr.start_processes(nop, df, gene_sequences, lsy, part, n, name)
|
||||
smg, sml, indexes = read_data_synteny(nop, name)
|
||||
sml = np.array(sml)
|
||||
smg = np.array(smg)
|
||||
indexes = np.array(indexes)
|
||||
return sml, smg, indexes
|
||||
|
||||
|
||||
def get_prediction(smg, sml, indexes, bls, blhs, dis, dps, dphs, model_name):
|
||||
preds = np.zeros((len(smg), 3))
|
||||
w = [0.86, 0.8, 0.06]
|
||||
for i in range(1, 4):
|
||||
try:
|
||||
model=tf.train.import_meta_graph(model_name+'_v'+str(i)+'/model.ckpt.meta')
|
||||
except:
|
||||
model = tf.train.import_meta_graph(
|
||||
model_name + '_v' + str(i) + '/model.ckpt.meta')
|
||||
except BaseException:
|
||||
print("Something wrong with the model.")
|
||||
continue
|
||||
with tf.Session() as sess:
|
||||
try:
|
||||
model.restore(sess,model_name+'_v'+str(i)+"/model.ckpt")
|
||||
model.restore(sess, model_name + '_v' + str(i) + "/model.ckpt")
|
||||
graph = tf.get_default_graph()
|
||||
synmgt,synmlt,blst,blhst,dpst,dphst,dist,lrt,yt=graph.get_collection("input_nodes")
|
||||
predictions=graph.get_tensor_by_name("Predictions/BiasAdd:0")
|
||||
synmgt, synmlt, blst, \
|
||||
blhst, dpst, dphst, \
|
||||
dist, lrt, yt = graph.get_collection("input_nodes")
|
||||
predictions = graph.get_tensor_by_name("Predictions/BiasAdd:0")
|
||||
print("Model Loaded Successfully :)")
|
||||
except:
|
||||
except BaseException:
|
||||
print(":(")
|
||||
sys.exit()
|
||||
|
||||
fd={synmgt:smg,
|
||||
synmlt:sml,
|
||||
blst:bls,
|
||||
blhst:blhs,
|
||||
dpst:dps.reshape((len(blhs),1)),
|
||||
dist:dis.reshape((len(blhs),1)),
|
||||
dphst:dphs.reshape((len(blhs),1))}
|
||||
preds_t_1=sess.run([predictions],feed_dict=fd)
|
||||
preds_t_1=np.array(preds_t_1)[0]
|
||||
fd={synmgt:smg.transpose((0,2,1,3)),
|
||||
synmlt:sml.transpose((0,2,1,3)),
|
||||
blst:blhs,
|
||||
blhst:bls,
|
||||
dpst:dphs.reshape((len(blhs),1)),
|
||||
dist:dis.reshape((len(blhs),1)),
|
||||
dphst:dps.reshape((len(blhs),1))}
|
||||
preds_t_2=sess.run([predictions],feed_dict=fd)
|
||||
preds_t_2=np.array(preds_t_2)[0]
|
||||
preds=preds+w[i-1]*(preds_t_1+preds_t_2)/2
|
||||
tf.reset_default_graph()
|
||||
preds=np.argmax(preds,axis=1)
|
||||
fd = {synmgt: smg,
|
||||
synmlt: sml,
|
||||
blst: bls,
|
||||
blhst: blhs,
|
||||
dpst: dps.reshape((len(blhs), 1)),
|
||||
dist: dis.reshape((len(blhs), 1)),
|
||||
dphst: dphs.reshape((len(blhs), 1))}
|
||||
preds_t_1 = sess.run([predictions], feed_dict=fd)
|
||||
preds_t_1 = np.array(preds_t_1)[0]
|
||||
fd = {synmgt: smg.transpose((0, 2, 1, 3)),
|
||||
synmlt: sml.transpose((0, 2, 1, 3)),
|
||||
blst: blhs,
|
||||
blhst: bls,
|
||||
dpst: dphs.reshape((len(blhs), 1)),
|
||||
dist: dis.reshape((len(blhs), 1)),
|
||||
dphst: dps.reshape((len(blhs), 1))}
|
||||
preds_t_2 = sess.run([predictions], feed_dict=fd)
|
||||
preds_t_2 = np.array(preds_t_2)[0]
|
||||
preds = preds + w[i - 1] * (preds_t_1 + preds_t_2) / 2
|
||||
tf.reset_default_graph()
|
||||
preds = np.argmax(preds, axis=1)
|
||||
print(preds.shape)
|
||||
return preds
|
||||
|
||||
def write_preds(fname,model_name,name,preds,index_dict,df):
|
||||
print("Writing predcitions to:","prediction_"+fname+"_"+model_name+"_"+name+"_multiple.txt")
|
||||
with open("prediction_"+fname+"_"+model_name+"_"+name+"_multiple.txt","w") as file:
|
||||
for index,row in progressbar.progressbar(df.iterrows()):
|
||||
|
||||
def write_preds(fname, model_name, name, preds, index_dict, df):
|
||||
print(
|
||||
"Writing predcitions to:",
|
||||
"prediction_" +
|
||||
fname +
|
||||
"_" +
|
||||
model_name +
|
||||
"_" +
|
||||
name +
|
||||
"_multiple.txt")
|
||||
with open("prediction_" + fname + "_" + model_name + "_" + name + "_multiple.txt", "w") as file:
|
||||
for index, row in progressbar.progressbar(df.iterrows()):
|
||||
file.write(str(row[0]))
|
||||
file.write("\t")
|
||||
file.write(row[1])
|
||||
|
|
@ -121,7 +143,7 @@ def write_preds(fname,model_name,name,preds,index_dict,df):
|
|||
if index in index_dict:
|
||||
file.write(str(preds[index_dict[index]]))
|
||||
file.write("\t")
|
||||
if preds[index_dict[index]]==row["label"]:
|
||||
if preds[index_dict[index]] == row["label"]:
|
||||
file.write(str(1))
|
||||
else:
|
||||
file.write(str(0))
|
||||
|
|
@ -131,31 +153,43 @@ def write_preds(fname,model_name,name,preds,index_dict,df):
|
|||
file.write("NaN")
|
||||
file.write("\n")
|
||||
|
||||
|
||||
def main():
|
||||
arg=sys.argv
|
||||
fname=arg[-6]
|
||||
model_name=arg[-5]
|
||||
nop=int(arg[-4])
|
||||
st=int(arg[-3])
|
||||
end=int(arg[-2])
|
||||
name=arg[-1]
|
||||
df=read_database(fname)
|
||||
df=select_data_by_length(df,st,end)
|
||||
n=3
|
||||
arg = sys.argv
|
||||
fname = arg[-6]
|
||||
model_name = arg[-5]
|
||||
nop = int(arg[-4])
|
||||
st = int(arg[-3])
|
||||
end = int(arg[-2])
|
||||
name = arg[-1]
|
||||
df = read_database(fname)
|
||||
df = select_data_by_length(df, st, end)
|
||||
n = 3
|
||||
|
||||
a,d,ld,ldg,cmap,cimap=read_genome_maps()#read the genome mapd
|
||||
a, d, ld, ldg, cmap, cimap = read_genome_maps() # read the genome mapd
|
||||
print("Genome Maps Loaded.")
|
||||
a_h=[df]
|
||||
d_h=["prediction"]
|
||||
a_h = [df]
|
||||
d_h = ["prediction"]
|
||||
|
||||
lsy,gene_sequences=create_synteny_features(a_h,d_h,n,a,d,ld,ldg,cmap,cimap,name)
|
||||
sml,smg,indexes=threadmaker(nop,df,lsy,gene_sequences,n,name)
|
||||
df_temp=df.loc[indexes]
|
||||
bls,blhs,dis,dps,dphs=create_tree_data("species_tree.tree",df_temp)
|
||||
index_dict=create_map_list(indexes)
|
||||
preds=get_prediction(smg,sml,indexes,bls,blhs,dis,dps,dphs,model_name)
|
||||
lsy, gene_sequences = create_synteny_features(
|
||||
a_h, d_h, n, a, d, ld, ldg, cmap, cimap, name)
|
||||
sml, smg, indexes = threadmaker(nop, df, lsy, gene_sequences, n, name)
|
||||
df_temp = df.loc[indexes]
|
||||
bls, blhs, dis, dps, dphs = create_tree_data("species_tree.tree", df_temp)
|
||||
index_dict = create_map_list(indexes)
|
||||
preds = get_prediction(
|
||||
smg,
|
||||
sml,
|
||||
indexes,
|
||||
bls,
|
||||
blhs,
|
||||
dis,
|
||||
dps,
|
||||
dphs,
|
||||
model_name)
|
||||
|
||||
write_preds(fname,model_name,name,preds,index_dict,df)
|
||||
write_preds(fname, model_name, name, preds, index_dict, df)
|
||||
|
||||
if __name__=="__main__":
|
||||
main()
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
import numpy as np
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
|
@ -9,77 +8,83 @@ from threads import Procerssrunner
|
|||
from read_get_gene_seq import read_gene_sequences
|
||||
from access_data_rest import update_rest
|
||||
|
||||
def read_data_synteny(nop,name):
|
||||
smg=[]
|
||||
sml=[]
|
||||
indexes=[]
|
||||
|
||||
def read_data_synteny(nop, name):
|
||||
smg = []
|
||||
sml = []
|
||||
indexes = []
|
||||
for i in range(nop):
|
||||
try:
|
||||
with open("temp_"+name+"/thread_"+str(i+1)+"_smg.temp","rb") as file:
|
||||
smg=smg+pickle.load(file)
|
||||
with open("temp_"+name+"/thread_"+str(i+1)+"_sml.temp","rb") as file:
|
||||
sml=sml+pickle.load(file)
|
||||
with open("temp_"+name+"/thread_"+str(i+1)+"_indexes.temp","rb") as file:
|
||||
indexes=indexes+pickle.load(file)
|
||||
with open("temp_" + name + "/thread_" + str(i + 1) + "_smg.temp", "rb") as file:
|
||||
smg = smg + pickle.load(file)
|
||||
with open("temp_" + name + "/thread_" + str(i + 1) + "_sml.temp", "rb") as file:
|
||||
sml = sml + pickle.load(file)
|
||||
with open("temp_" + name + "/thread_" + str(i + 1) + "_indexes.temp", "rb") as file:
|
||||
indexes = indexes + pickle.load(file)
|
||||
except Exception as e:
|
||||
print("Problem with thread",i+1,"detected for",name,e)
|
||||
print("Problem with thread", i + 1, "detected for", name, e)
|
||||
continue
|
||||
print(len(indexes))
|
||||
return smg,sml,indexes
|
||||
return smg, sml, indexes
|
||||
|
||||
|
||||
def load_neighbor_genes():
|
||||
with open("processed/neighbor_genes.json","r") as file:
|
||||
lsy=dict(json.load(file))
|
||||
with open("processed/neighbor_genes.json", "r") as file:
|
||||
lsy = dict(json.load(file))
|
||||
print(len(lsy))
|
||||
print("Neighbor Genes Loaded")
|
||||
return lsy
|
||||
|
||||
|
||||
def read_data_homology(dirname):
|
||||
lf=os.listdir(dirname)
|
||||
if len(lf)==0:
|
||||
lf = os.listdir(dirname)
|
||||
if len(lf) == 0:
|
||||
print("No Files in the Directory!!!!!!!")
|
||||
sys.exit(1)
|
||||
a_h=[]
|
||||
d_h=[]
|
||||
a_h = []
|
||||
d_h = []
|
||||
for x in lf:
|
||||
df,n=read_db_homology(dirname,x)
|
||||
n=n.split()[0]
|
||||
df, n = read_db_homology(dirname, x)
|
||||
n = n.split()[0]
|
||||
try:
|
||||
indexes=np.load("processed/"+n+"_selected_indexes.npy")
|
||||
except:
|
||||
print("Incomplete data for:",n)
|
||||
df=df.loc[indexes]
|
||||
indexes = np.load("processed/" + n + "_selected_indexes.npy")
|
||||
except BaseException:
|
||||
print("Incomplete data for:", n)
|
||||
df = df.loc[indexes]
|
||||
a_h.append(df)
|
||||
d_h.append(n)
|
||||
return a_h,d_h
|
||||
return a_h, d_h
|
||||
|
||||
|
||||
def main():
|
||||
arg=sys.argv
|
||||
nop=int(arg[-1])
|
||||
n=3
|
||||
a_h,d_h=read_data_homology("data_homology")
|
||||
arg = sys.argv
|
||||
nop = int(arg[-1])
|
||||
n = 3
|
||||
a_h, d_h = read_data_homology("data_homology")
|
||||
print("Data Read")
|
||||
lsy=load_neighbor_genes()
|
||||
gene_sequences=read_gene_sequences(a_h,lsy,"geneseq","gene_seq_positive")
|
||||
gene_sequences=update_rest(gene_sequences,"gene_seq_positive")
|
||||
lsy = load_neighbor_genes()
|
||||
gene_sequences = read_gene_sequences(
|
||||
a_h, lsy, "geneseq", "gene_seq_positive")
|
||||
gene_sequences = update_rest(gene_sequences, "gene_seq_positive")
|
||||
print("Gene Sequences Loaded.")
|
||||
if not os.path.isdir("processed/synteny_matrices"):
|
||||
os.mkdir("processed/synteny_matrices")
|
||||
ndir="processed/synteny_matrices/"
|
||||
nf1="synteny_matrices_global"
|
||||
nf2="synteny_matrices_local"
|
||||
nf3="indexes"
|
||||
ndir = "processed/synteny_matrices/"
|
||||
nf1 = "synteny_matrices_global"
|
||||
nf2 = "synteny_matrices_local"
|
||||
nf3 = "indexes"
|
||||
for i in range(len(a_h)):
|
||||
df=a_h[i]
|
||||
part=len(df)//nop
|
||||
pr=Procerssrunner()
|
||||
pr.start_processes(nop,df,gene_sequences,lsy,part,n,d_h[i])
|
||||
smg,sml,indexes=read_data_synteny(nop,d_h[i])
|
||||
df = a_h[i]
|
||||
part = len(df) // nop
|
||||
pr = Procerssrunner()
|
||||
pr.start_processes(nop, df, gene_sequences, lsy, part, n, d_h[i])
|
||||
smg, sml, indexes = read_data_synteny(nop, d_h[i])
|
||||
print(len(indexes))
|
||||
np.save(ndir+str(d_h[i])+"_"+nf1,smg)
|
||||
np.save(ndir+str(d_h[i])+"_"+nf2,sml)
|
||||
np.save(ndir+str(d_h[i])+"_"+nf3,indexes)
|
||||
np.save(ndir + str(d_h[i]) + "_" + nf1, smg)
|
||||
np.save(ndir + str(d_h[i]) + "_" + nf2, sml)
|
||||
np.save(ndir + str(d_h[i]) + "_" + nf3, indexes)
|
||||
print("Synteny Matrices Created Successfully :)")
|
||||
|
||||
if __name__=="__main__":
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
|
|||
234
process_data.py
234
process_data.py
|
|
@ -1,154 +1,166 @@
|
|||
import pandas
|
||||
import gc
|
||||
import numpy as np
|
||||
import json
|
||||
import os
|
||||
import progressbar
|
||||
from save_data import write_dict_json
|
||||
|
||||
def create_map_list(l): #this function maps the indexes to values
|
||||
t={}
|
||||
|
||||
def create_map_list(l): # this function maps the indexes to values
|
||||
t = {}
|
||||
for i in range(len(l)):
|
||||
t[l[i]]=i
|
||||
t[l[i]] = i
|
||||
return t
|
||||
|
||||
def create_chromosome_maps(a,n):
|
||||
cmap=[]
|
||||
cimap=[]
|
||||
|
||||
def create_chromosome_maps(a, n):
|
||||
cmap = []
|
||||
cimap = []
|
||||
for df in progressbar.progressbar(a):
|
||||
chmap={}
|
||||
chindmap={}
|
||||
for index,row in df.iterrows():
|
||||
g=row.gene_id
|
||||
chmap = {}
|
||||
chindmap = {}
|
||||
for index, row in df.iterrows():
|
||||
g = row.gene_id
|
||||
try:
|
||||
temp=chmap[g]
|
||||
except:
|
||||
chmap[g]=str(row.Chr)
|
||||
_ = chmap[g]
|
||||
except BaseException:
|
||||
chmap[g] = str(row.Chr)
|
||||
if str(row.Chr) in chindmap:
|
||||
chindmap[str(row.Chr)].append(index)
|
||||
else:
|
||||
chindmap[str(row.Chr)]=[]
|
||||
chindmap[str(row.Chr)] = []
|
||||
chindmap[str(row.Chr)].append(index)
|
||||
cmap.append(chmap)
|
||||
cimap.append(chindmap)
|
||||
return cmap,cimap
|
||||
return cmap, cimap
|
||||
|
||||
def list_dict_genomes(a,n):
|
||||
lst=[]
|
||||
ldt=[]
|
||||
|
||||
def list_dict_genomes(a, n):
|
||||
lst = []
|
||||
ldt = []
|
||||
for x in a:
|
||||
ldgt={}
|
||||
uc=list(x["gene_id"])
|
||||
for i,r in x.iterrows():
|
||||
ldgt[r.gene_id]=i
|
||||
ldgt = {}
|
||||
uc = list(x["gene_id"])
|
||||
for i, r in x.iterrows():
|
||||
ldgt[r.gene_id] = i
|
||||
lst.append(uc)
|
||||
ldt.append(ldgt)
|
||||
return lst,ldt
|
||||
return lst, ldt
|
||||
|
||||
def get_nearest_neighbors(g,gs,n,a,d,ld,ldg,cmap,cimap):
|
||||
#print("Finding Neighbor Genes")
|
||||
ne=[] #list to store the backward genes
|
||||
nr=[] #list to store the forward genes
|
||||
gi=d[gs.capitalize()] #get the address of the corresponding species to which the gene belongs whose neighbor has to be found
|
||||
sldf=a[gi]#select the dataframe
|
||||
scmap=cmap[gi]#select the correct chromosome map
|
||||
scimap=cimap[gi]#select the correct index maps
|
||||
|
||||
def get_nearest_neighbors(g, gs, n, a, d, ld, ldg, cmap, cimap):
|
||||
# print("Finding Neighbor Genes")
|
||||
ne = [] # list to store the backward genes
|
||||
nr = [] # list to store the forward genes
|
||||
# get the address of the corresponding species to which the gene belongs
|
||||
# whose neighbor has to be found
|
||||
gi = d[gs.capitalize()]
|
||||
sldf = a[gi] # select the dataframe
|
||||
scmap = cmap[gi] # select the correct chromosome map
|
||||
scimap = cimap[gi] # select the correct index maps
|
||||
try:
|
||||
sld=ld[gi]#see if the corresponding gene map exists
|
||||
except:
|
||||
#print("Length of Dataframes:{} \t Length of Loaded Genes:{} \t Length of Loaded Genomes Dictionaries:{}".format(len(a),len(ld),len(ldg)))
|
||||
return ne,nr
|
||||
sldg=ldg[gi]#select the corresponding map
|
||||
if g not in sldg:#if the gene is not present in the dataframe return empty lists
|
||||
#print(g,"\t",gs)
|
||||
return ne,nr
|
||||
i=sldg[g]#find the index of the gene
|
||||
chromosome_id=scmap[g]#get the chromosome no from the database.
|
||||
scimap=scimap[chromosome_id]#select the correct chromosomes indexes
|
||||
sldf=sldf.loc[scimap]#select only the same chromosome genes.
|
||||
#get the -n neighbors
|
||||
start=int(sldf.loc[i]['start'])#get the start location of the gene
|
||||
flag=0
|
||||
_ = ld[gi] # see if the corresponding gene map exists
|
||||
except BaseException:
|
||||
# print("Length of Dataframes:{} \t Length of Loaded Genes:{} \t Length of Loaded Genomes Dictionaries:{}".format(len(a),len(ld),len(ldg)))
|
||||
return ne, nr
|
||||
sldg = ldg[gi] # select the corresponding map
|
||||
if g not in sldg: # if the gene is not present in the dataframe return empty lists
|
||||
# print(g,"\t",gs)
|
||||
return ne, nr
|
||||
i = sldg[g] # find the index of the gene
|
||||
chromosome_id = scmap[g] # get the chromosome no from the database.
|
||||
scimap = scimap[chromosome_id] # select the correct chromosomes indexes
|
||||
sldf = sldf.loc[scimap] # select only the same chromosome genes.
|
||||
# get the -n neighbors
|
||||
start = int(sldf.loc[i]['start']) # get the start location of the gene
|
||||
flag = 0
|
||||
for j in range(n):
|
||||
if flag==1:
|
||||
if flag == 1:
|
||||
ne.append("NULL_GENE")
|
||||
continue
|
||||
itemp=0
|
||||
#select the column
|
||||
end=list(sldf.end)
|
||||
end=np.array(end)
|
||||
assert(len(end)==len(sldf))
|
||||
end=end-start #subtract start from it so as to get relative position
|
||||
end_s=np.argsort(end)#sort them by the order of distance
|
||||
if end[end_s[0]]>=0:#if all the genes end ahead of the one in consideration
|
||||
flag=1#increment the pointer
|
||||
ne.append("NULL_GENE")#append the NULL_GENE value
|
||||
itemp = 0
|
||||
# select the column
|
||||
end = list(sldf.end)
|
||||
end = np.array(end)
|
||||
assert(len(end) == len(sldf))
|
||||
end = end - start # subtract start from it so as to get relative position
|
||||
end_s = np.argsort(end) # sort them by the order of distance
|
||||
if end[end_s[0]] >= 0: # if all the genes end ahead of the one in consideration
|
||||
flag = 1 # increment the pointer
|
||||
ne.append("NULL_GENE") # append the NULL_GENE value
|
||||
continue
|
||||
for k in end_s:#iterate through the sorted array
|
||||
if end[k]<0 and end[k+1]>=0:#find the first value that is negative and the next one is positive to get the nearest gene
|
||||
itemp=k
|
||||
for k in end_s: # iterate through the sorted array
|
||||
# find the first value that is negative and the next one is
|
||||
# positive to get the nearest gene
|
||||
if end[k] < 0 and end[k + 1] >= 0:
|
||||
itemp = k
|
||||
break
|
||||
itemp=scimap[itemp]
|
||||
itemp = scimap[itemp]
|
||||
ne.append(sldf.loc[itemp].gene_id)
|
||||
start=int(sldf.loc[itemp].start)#make "start" the start location of the current gene
|
||||
#print(start)
|
||||
#get the +n neighbors
|
||||
flag=0
|
||||
end=int(sldf.loc[i].end)
|
||||
# make "start" the start location of the current gene
|
||||
start = int(sldf.loc[itemp].start)
|
||||
# print(start)
|
||||
# get the +n neighbors
|
||||
flag = 0
|
||||
end = int(sldf.loc[i].end)
|
||||
for j in range(n):
|
||||
if flag==1:
|
||||
if flag == 1:
|
||||
nr.append("NULL_GENE")
|
||||
continue
|
||||
itemp=0
|
||||
start=list(sldf.start)
|
||||
start=np.array(start)
|
||||
start=start-end
|
||||
start_s=np.argsort(start)
|
||||
if start[start_s[-1]]<0:
|
||||
flag=1
|
||||
itemp = 0
|
||||
start = list(sldf.start)
|
||||
start = np.array(start)
|
||||
start = start - end
|
||||
start_s = np.argsort(start)
|
||||
if start[start_s[-1]] < 0:
|
||||
flag = 1
|
||||
nr.append("NULL_GENE")
|
||||
continue
|
||||
for k in start_s:
|
||||
if start[k]>0:
|
||||
itemp=k
|
||||
if start[k] > 0:
|
||||
itemp = k
|
||||
break
|
||||
itemp=scimap[itemp]
|
||||
itemp = scimap[itemp]
|
||||
nr.append(sldf.loc[itemp].gene_id)
|
||||
end=int(sldf.loc[itemp].end)
|
||||
return ne,nr
|
||||
end = int(sldf.loc[itemp].end)
|
||||
return ne, nr
|
||||
|
||||
def create_data_homology_ls(a_h,d_h,n,a,d,ld,ldg,cmap,cimap,to_write):
|
||||
lsy={} #dictionary which stores +/- n genes of the given gene by id. Each key is a gene id which corresponds to the one in center.
|
||||
lsytemp={}
|
||||
name="neighbor_genes"
|
||||
|
||||
def create_data_homology_ls(a_h, d_h, n, a, d, ld, ldg, cmap, cimap, to_write):
|
||||
# dictionary which stores +/- n genes of the given gene by id. Each key is
|
||||
# a gene id which corresponds to the one in center.
|
||||
lsy = {}
|
||||
lsytemp = {}
|
||||
name = "neighbor_genes"
|
||||
for df in a_h:
|
||||
for _,row in progressbar.progressbar(df.iterrows()):
|
||||
x=row["gene_stable_id"]
|
||||
y=row["homology_gene_stable_id"]
|
||||
xs=row["species"]
|
||||
ys=row["homology_species"]
|
||||
for _, row in progressbar.progressbar(df.iterrows()):
|
||||
x = row["gene_stable_id"]
|
||||
y = row["homology_gene_stable_id"]
|
||||
xs = row["species"]
|
||||
ys = row["homology_species"]
|
||||
try:
|
||||
z=lsy[x]
|
||||
except:
|
||||
_ = lsy[x]
|
||||
except BaseException:
|
||||
try:
|
||||
t2=d[xs.capitalize()]#see if the species exist in genomic maps
|
||||
xl,xr=get_nearest_neighbors(x,xs,n,a,d,ld,ldg,cmap,cimap)
|
||||
if len(xl)!=0:#check if neighboring genes were successfully found
|
||||
lsy[x]=dict(b=xl,f=xr)
|
||||
lsytemp[x]=dict(b=xl,f=xr)
|
||||
except:
|
||||
# see if the species exist in genomic maps
|
||||
_ = d[xs.capitalize()]
|
||||
xl, xr = get_nearest_neighbors(
|
||||
x, xs, n, a, d, ld, ldg, cmap, cimap)
|
||||
if len(
|
||||
xl) != 0: # check if neighboring genes were successfully found
|
||||
lsy[x] = dict(b=xl, f=xr)
|
||||
lsytemp[x] = dict(b=xl, f=xr)
|
||||
except BaseException:
|
||||
continue
|
||||
try:
|
||||
z=lsy[y]
|
||||
except:
|
||||
_ = lsy[y]
|
||||
except BaseException:
|
||||
try:
|
||||
t2=d[ys.capitalize()]
|
||||
yl,yr=get_nearest_neighbors(y,ys,n,a,d,ld,ldg,cmap,cimap)
|
||||
if len(yl)!=0:
|
||||
lsy[y]=dict(b=yl,f=yr)
|
||||
lsytemp[y]=dict(b=yl,f=yr)
|
||||
except:
|
||||
_ = d[ys.capitalize()]
|
||||
yl, yr = get_nearest_neighbors(
|
||||
y, ys, n, a, d, ld, ldg, cmap, cimap)
|
||||
if len(yl) != 0:
|
||||
lsy[y] = dict(b=yl, f=yr)
|
||||
lsytemp[y] = dict(b=yl, f=yr)
|
||||
except BaseException:
|
||||
continue
|
||||
if to_write==1:
|
||||
write_dict_json(name,"processed",lsy)
|
||||
return lsy
|
||||
if to_write == 1:
|
||||
write_dict_json(name, "processed", lsy)
|
||||
return lsy
|
||||
|
|
|
|||
|
|
@ -1,9 +1,5 @@
|
|||
import pandas as pd
|
||||
import numpy as np
|
||||
import os
|
||||
import sys
|
||||
import progressbar
|
||||
import json
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
import sys
|
||||
from neighbor_genes import read_genome_maps
|
||||
from process_data import create_data_homology_ls
|
||||
|
|
@ -13,45 +9,55 @@ from access_data_rest import update_rest
|
|||
from prepare_synteny_matrix import read_data_synteny
|
||||
from save_data import write_dict_json
|
||||
|
||||
|
||||
def read_database_txt(filename):
|
||||
df=pd.read_csv(filename,sep="\t",header=None)
|
||||
df=df.drop(0,axis=1)
|
||||
df.columns=["gene_stable_id","species","homology_gene_stable_id","homology_species","wga","goc","homology_type"]
|
||||
df = pd.read_csv(filename, sep="\t", header=None)
|
||||
df = df.drop(0, axis=1)
|
||||
df.columns = [
|
||||
"gene_stable_id",
|
||||
"species",
|
||||
"homology_gene_stable_id",
|
||||
"homology_species",
|
||||
"wga",
|
||||
"goc",
|
||||
"homology_type"]
|
||||
return df
|
||||
|
||||
|
||||
def main():
|
||||
arg=sys.argv
|
||||
a,d,ld,ldg,cmap,cimap=read_genome_maps()
|
||||
arg = sys.argv
|
||||
a, d, ld, ldg, cmap, cimap = read_genome_maps()
|
||||
print("Genome Maps Loaded.")
|
||||
df=read_database_txt(arg[-2])
|
||||
nop=int(arg[-1])
|
||||
df = read_database_txt(arg[-2])
|
||||
nop = int(arg[-1])
|
||||
print("Data Read.")
|
||||
a_h=[]
|
||||
d_h=[]
|
||||
a_h = []
|
||||
d_h = []
|
||||
a_h.append(df)
|
||||
d_h.append(arg[-2].split(".")[0])
|
||||
n=3
|
||||
lsy=create_data_homology_ls(a_h,d_h,n,a,d,ld,ldg,cmap,cimap,0)
|
||||
write_dict_json("neighbor_genes_negative","processed",lsy)
|
||||
n = 3
|
||||
lsy = create_data_homology_ls(a_h, d_h, n, a, d, ld, ldg, cmap, cimap, 0)
|
||||
write_dict_json("neighbor_genes_negative", "processed", lsy)
|
||||
print("Neighbor Genes Found and Saved Successfully:)")
|
||||
gene_sequences=read_gene_sequences(a_h,lsy,"geneseq","gene_seq_negative")
|
||||
gene_sequences=update_rest(gene_sequences,"gene_seq_negative")
|
||||
ndir="processed/synteny_matrices/"
|
||||
nf1="synteny_matrices_global"
|
||||
nf2="synteny_matrices_local"
|
||||
nf3="indexes"
|
||||
gene_sequences = read_gene_sequences(
|
||||
a_h, lsy, "geneseq", "gene_seq_negative")
|
||||
gene_sequences = update_rest(gene_sequences, "gene_seq_negative")
|
||||
ndir = "processed/synteny_matrices/"
|
||||
nf1 = "synteny_matrices_global"
|
||||
nf2 = "synteny_matrices_local"
|
||||
nf3 = "indexes"
|
||||
for i in range(len(a_h)):
|
||||
df=a_h[i]
|
||||
part=len(df)//nop
|
||||
pr=Procerssrunner()
|
||||
pr.start_processes(nop,df,gene_sequences,lsy,part,n,d_h[i])
|
||||
smg,sml,indexes=read_data_synteny(nop,d_h[i])
|
||||
df = a_h[i]
|
||||
part = len(df) // nop
|
||||
pr = Procerssrunner()
|
||||
pr.start_processes(nop, df, gene_sequences, lsy, part, n, d_h[i])
|
||||
smg, sml, indexes = read_data_synteny(nop, d_h[i])
|
||||
print(len(indexes))
|
||||
np.save(ndir+str(d_h[i])+"_"+nf1,smg)
|
||||
np.save(ndir+str(d_h[i])+"_"+nf2,sml)
|
||||
np.save(ndir+str(d_h[i])+"_"+nf3,indexes)
|
||||
np.save(ndir + str(d_h[i]) + "_" + nf1, smg)
|
||||
np.save(ndir + str(d_h[i]) + "_" + nf2, sml)
|
||||
np.save(ndir + str(d_h[i]) + "_" + nf3, indexes)
|
||||
print("Synteny Matrices Created Successfully :)")
|
||||
|
||||
if __name__=="__main__":
|
||||
main()
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
|
|||
80
read_data.py
80
read_data.py
|
|
@ -1,50 +1,72 @@
|
|||
import os
|
||||
import pandas as pd
|
||||
import gzip
|
||||
import sys
|
||||
import progressbar
|
||||
import traceback
|
||||
|
||||
|
||||
def clear_data(x):
|
||||
if x==None:
|
||||
if x is None:
|
||||
return x
|
||||
x=x.split()
|
||||
x = x.split()
|
||||
try:
|
||||
x=x[1]
|
||||
except:
|
||||
c=0
|
||||
#print(x)
|
||||
x=x[1:-1]
|
||||
x = x[1]
|
||||
except BaseException:
|
||||
_ = 0
|
||||
# print(x)
|
||||
x = x[1:-1]
|
||||
return x
|
||||
|
||||
def read_data_genome(dir_name,a,dict_ind_genome):
|
||||
lf=os.listdir(dir_name)
|
||||
if len(lf)==0:
|
||||
|
||||
def read_data_genome(dir_name, a, dict_ind_genome):
|
||||
lf = os.listdir(dir_name)
|
||||
if len(lf) == 0:
|
||||
print("No files in the data directory!!!!!!")
|
||||
sys.exit(1)
|
||||
colname=["Chr","source","feature","start","end","score","strand","frame","attribute"]
|
||||
colname = [
|
||||
"Chr",
|
||||
"source",
|
||||
"feature",
|
||||
"start",
|
||||
"end",
|
||||
"score",
|
||||
"strand",
|
||||
"frame",
|
||||
"attribute"]
|
||||
print("Going to read data:")
|
||||
for x in progressbar.progressbar(range(len(lf))):
|
||||
data_gene=pd.read_csv(dir_name+"/"+lf[x],compression='gzip',sep='\t',comment='#',header=None,names=colname)
|
||||
#print(data_gene.head)
|
||||
data_gene=data_gene[data_gene["feature"]=="gene"]
|
||||
tmp=data_gene["attribute"].str.split(";",expand=True)
|
||||
tmp=tmp.iloc[:,:5]
|
||||
data_gene[["gene_id","gene_version","gene_name","gene_source","gene_biotype"]]=tmp
|
||||
data_gene=data_gene.drop("attribute",axis=1)
|
||||
#print(data_gene[0:10])
|
||||
data_gene = pd.read_csv(
|
||||
dir_name + "/" + lf[x],
|
||||
compression='gzip',
|
||||
sep='\t',
|
||||
comment='#',
|
||||
header=None,
|
||||
names=colname)
|
||||
# print(data_gene.head)
|
||||
data_gene = data_gene[data_gene["feature"] == "gene"]
|
||||
tmp = data_gene["attribute"].str.split(";", expand=True)
|
||||
tmp = tmp.iloc[:, :5]
|
||||
data_gene[["gene_id", "gene_version", "gene_name",
|
||||
"gene_source", "gene_biotype"]] = tmp
|
||||
data_gene = data_gene.drop("attribute", axis=1)
|
||||
# print(data_gene[0:10])
|
||||
try:
|
||||
for y in ["gene_version","gene_name","gene_source","gene_biotype","gene_id"]:
|
||||
data_gene[y]=data_gene[y].apply(clear_data)
|
||||
for y in [
|
||||
"gene_version",
|
||||
"gene_name",
|
||||
"gene_source",
|
||||
"gene_biotype",
|
||||
"gene_id"]:
|
||||
data_gene[y] = data_gene[y].apply(clear_data)
|
||||
except Exception as e:
|
||||
traceback.print_exc()
|
||||
print(e)
|
||||
continue
|
||||
#print(data_gene[0:10])
|
||||
data_gene=data_gene[(data_gene['gene_biotype']=='protein_coding') | (data_gene['gene_source']=='protein_coding')]
|
||||
#print(data_gene[data_gene["gene_id"]=="ENSNGAG00000000407"])
|
||||
# print(data_gene[0:10])
|
||||
data_gene = data_gene[(data_gene['gene_biotype'] == 'protein_coding') | (
|
||||
data_gene['gene_source'] == 'protein_coding')]
|
||||
# print(data_gene[data_gene["gene_id"]=="ENSNGAG00000000407"])
|
||||
a.append(data_gene)
|
||||
n=lf[x].split(".")[0]
|
||||
dict_ind_genome[n]=len(a)-1
|
||||
return a,dict_ind_genome
|
||||
|
||||
n = lf[x].split(".")[0]
|
||||
dict_ind_genome[n] = len(a) - 1
|
||||
return a, dict_ind_genome
|
||||
|
|
|
|||
|
|
@ -1,121 +1,132 @@
|
|||
import json
|
||||
from Bio import SeqIO
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import os
|
||||
import gzip
|
||||
import progressbar
|
||||
|
||||
|
||||
def read_from_multiple_lsy(lsyfl):
|
||||
lsy={}
|
||||
lsy = {}
|
||||
for f in lsyfl:
|
||||
d={}
|
||||
with open(f,"r") as file:
|
||||
d=dict(json.load(file))
|
||||
d = {}
|
||||
with open(f, "r") as file:
|
||||
d = dict(json.load(file))
|
||||
for t in d:
|
||||
lsy[t]=d[t]
|
||||
lsy[t] = d[t]
|
||||
return lsy
|
||||
|
||||
#this function updates the given dictionary with the given keys and values list
|
||||
def create_dict(keys,values,dictionary):
|
||||
# this function updates the given dictionary with the given keys and
|
||||
# values list
|
||||
|
||||
|
||||
def create_dict(keys, values, dictionary):
|
||||
for i in range(len(keys)):
|
||||
if keys[i] not in dictionary:
|
||||
dictionary[keys[i]]=values[i]
|
||||
dictionary[keys[i]] = values[i]
|
||||
|
||||
return dictionary
|
||||
|
||||
#this function maps all the genes to their respective species.
|
||||
#(Function: when finding the species of any gene we do not need to search the entire dataframe)
|
||||
def group_seq_by_species(df,g_to_sp):
|
||||
sph=list(df.homology_species)
|
||||
ghsp=list(df.homology_gene_stable_id)
|
||||
sp=list(df.species)
|
||||
gsp=list(df.gene_stable_id)
|
||||
create_dict(gsp,sp,g_to_sp)
|
||||
create_dict(ghsp,sph,g_to_sp)
|
||||
# this function maps all the genes to their respective species.
|
||||
# (Function: when finding the species of any gene we do not need to search the entire dataframe)
|
||||
|
||||
|
||||
def group_seq_by_species(df, g_to_sp):
|
||||
sph = list(df.homology_species)
|
||||
ghsp = list(df.homology_gene_stable_id)
|
||||
sp = list(df.species)
|
||||
gsp = list(df.gene_stable_id)
|
||||
create_dict(gsp, sp, g_to_sp)
|
||||
create_dict(ghsp, sph, g_to_sp)
|
||||
return g_to_sp
|
||||
|
||||
#this function returns the gene-id and gene-biotype from the description in the fasta file record.
|
||||
# this function returns the gene-id and gene-biotype from the description
|
||||
# in the fasta file record.
|
||||
|
||||
|
||||
def description_cleaner(description):
|
||||
description=description.split()
|
||||
t=""
|
||||
gbt=""
|
||||
description = description.split()
|
||||
t = ""
|
||||
gbt = ""
|
||||
for x in description:
|
||||
try:
|
||||
x=x.split(":")
|
||||
if x[0]=="gene":
|
||||
t=x[1].split(".")[0]
|
||||
if x[0]=="gene_biotype":
|
||||
gbt=x[1]
|
||||
except:
|
||||
return "aa","aa"
|
||||
return t,gbt
|
||||
x = x.split(":")
|
||||
if x[0] == "gene":
|
||||
t = x[1].split(".")[0]
|
||||
if x[0] == "gene_biotype":
|
||||
gbt = x[1]
|
||||
except BaseException:
|
||||
return "aa", "aa"
|
||||
return t, gbt
|
||||
|
||||
def read_gene_seq(dirname,s,genes_by_species):
|
||||
lof=os.listdir(dirname)#list all the files in the sequences directory
|
||||
ftr=[]
|
||||
|
||||
def read_gene_seq(dirname, s, genes_by_species):
|
||||
lof = os.listdir(dirname) # list all the files in the sequences directory
|
||||
ftr = []
|
||||
for f in lof:
|
||||
if f.split(".")[0] in s:#check whether the species is present in the species to read list. Will skip those species which are not present in the dataframe
|
||||
if f.split(".")[
|
||||
0] in s: # check whether the species is present in the species to read list. Will skip those species which are not present in the dataframe
|
||||
ftr.append(f)
|
||||
data={}
|
||||
data = {}
|
||||
for f in progressbar.progressbar(ftr):
|
||||
species=f.split(".")[0].lower()
|
||||
with gzip.open(dirname+"/"+f,"rt") as file:
|
||||
record=SeqIO.parse(file,"fasta")
|
||||
species = f.split(".")[0].lower()
|
||||
with gzip.open(dirname + "/" + f, "rt") as file:
|
||||
record = SeqIO.parse(file, "fasta")
|
||||
for r in record:
|
||||
gid,gbt=description_cleaner(r.description)
|
||||
if str(gid) not in data and str(gid) in genes_by_species[species] and gbt=="protein_coding":
|
||||
data[gid]=str(r.seq)
|
||||
gid, gbt = description_cleaner(r.description)
|
||||
if str(gid) not in data and str(
|
||||
gid) in genes_by_species[species] and gbt == "protein_coding":
|
||||
data[gid] = str(r.seq)
|
||||
return data
|
||||
|
||||
def read_gene_sequences(hdf,lsy,data_dir,fname):
|
||||
|
||||
def read_gene_sequences(hdf, lsy, data_dir, fname):
|
||||
"""The basic idea here is to create a list/dictionary of all the genes by their species.
|
||||
Once the mapping is done, all the respective fasta sequence files are read by Species
|
||||
and the CDNA sequences for each gene in the species record are read and stored.
|
||||
Thus we don't have to read the same file multiple times."""
|
||||
grouped_genes={}
|
||||
gene_by_species_dict={}
|
||||
grouped_genes = {}
|
||||
gene_by_species_dict = {}
|
||||
for df in progressbar.progressbar(hdf):
|
||||
grouped_genes=group_seq_by_species(df,grouped_genes)
|
||||
grouped_genes = group_seq_by_species(df, grouped_genes)
|
||||
for i in df.homology_species.unique():
|
||||
gene_by_species_dict[i]=[]
|
||||
gene_by_species_dict[i] = []
|
||||
for i in df.species.unique():
|
||||
gene_by_species_dict[i]=[]
|
||||
|
||||
gene_by_species_dict[i] = []
|
||||
|
||||
for x in progressbar.progressbar(lsy):
|
||||
try:
|
||||
species=grouped_genes[x]#get the species
|
||||
except:
|
||||
species = grouped_genes[x] # get the species
|
||||
except BaseException:
|
||||
continue
|
||||
if x not in gene_by_species_dict[species]:#check if the gene already exists in the species dict or not.
|
||||
# check if the gene already exists in the species dict or not.
|
||||
if x not in gene_by_species_dict[species]:
|
||||
gene_by_species_dict[species].append(x)
|
||||
xl=lsy[x]['b']
|
||||
xr=lsy[x]['f']
|
||||
xl = lsy[x]['b']
|
||||
xr = lsy[x]['f']
|
||||
for gxl in xl:
|
||||
if gxl=="NULL_GENE":
|
||||
if gxl == "NULL_GENE":
|
||||
break
|
||||
if gxl not in gene_by_species_dict[species]:
|
||||
gene_by_species_dict[species].append(gxl)
|
||||
for gxr in xr:
|
||||
if gxr=="NULL_GENE":
|
||||
if gxr == "NULL_GENE":
|
||||
break
|
||||
if gxr not in gene_by_species_dict[species]:
|
||||
gene_by_species_dict[species].append(gxr)
|
||||
|
||||
|
||||
s=[x for x in gene_by_species_dict if len(gene_by_species_dict[x])!=0]#select those species only whose gene sequences we have to read.
|
||||
s=[x.capitalize() for x in s]
|
||||
|
||||
data=read_gene_seq(data_dir,s,gene_by_species_dict)
|
||||
not_found={}
|
||||
# select those species only whose gene sequences we have to read.
|
||||
s = [x for x in gene_by_species_dict if len(gene_by_species_dict[x]) != 0]
|
||||
s = [x.capitalize() for x in s]
|
||||
|
||||
data = read_gene_seq(data_dir, s, gene_by_species_dict)
|
||||
not_found = {}
|
||||
for species in gene_by_species_dict:
|
||||
for gene in gene_by_species_dict[species]:
|
||||
try:
|
||||
_=data[gene]
|
||||
except:
|
||||
not_found[gene]=1
|
||||
_ = data[gene]
|
||||
except BaseException:
|
||||
not_found[gene] = 1
|
||||
|
||||
with open("processed/not_found_"+fname+".json","w") as file:
|
||||
json.dump(not_found,file)
|
||||
with open("processed/not_found_" + fname + ".json", "w") as file:
|
||||
json.dump(not_found, file)
|
||||
return data
|
||||
|
|
|
|||
31
save_data.py
31
save_data.py
|
|
@ -1,22 +1,23 @@
|
|||
import os
|
||||
import pickle
|
||||
import json
|
||||
import sys
|
||||
|
||||
def write_dict_json(name,dir,d):
|
||||
|
||||
def write_dict_json(name, dir, d):
|
||||
if not os.path.exists(dir):
|
||||
os.mkdir(dir)
|
||||
|
||||
path=os.path.join(dir,name+".json")
|
||||
with open(path,'w') as file:
|
||||
json.dump(d,file)
|
||||
|
||||
def write_data_synteny(smg,sml,indexes,i,name):
|
||||
if not os.path.exists("temp_"+name):
|
||||
os.mkdir("temp_"+name)
|
||||
with open("temp_"+name+"/thread_"+str(i+1)+"_smg.temp","wb") as file:
|
||||
pickle.dump(smg,file)
|
||||
with open("temp_"+name+"/thread_"+str(i+1)+"_sml.temp","wb") as file:
|
||||
pickle.dump(sml,file)
|
||||
with open("temp_"+name+"/thread_"+str(i+1)+"_indexes.temp","wb") as file:
|
||||
pickle.dump(indexes,file)
|
||||
path = os.path.join(dir, name + ".json")
|
||||
with open(path, 'w') as file:
|
||||
json.dump(d, file)
|
||||
|
||||
|
||||
def write_data_synteny(smg, sml, indexes, i, name):
|
||||
if not os.path.exists("temp_" + name):
|
||||
os.mkdir("temp_" + name)
|
||||
with open("temp_" + name + "/thread_" + str(i + 1) + "_smg.temp", "wb") as file:
|
||||
pickle.dump(smg, file)
|
||||
with open("temp_" + name + "/thread_" + str(i + 1) + "_sml.temp", "wb") as file:
|
||||
pickle.dump(sml, file)
|
||||
with open("temp_" + name + "/thread_" + str(i + 1) + "_indexes.temp", "wb") as file:
|
||||
pickle.dump(indexes, file)
|
||||
|
|
|
|||
|
|
@ -1,50 +1,52 @@
|
|||
import pandas as pd
|
||||
import gc
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
import json
|
||||
import os
|
||||
import progressbar
|
||||
import sys
|
||||
from selector import select,create_map_reverse
|
||||
from selector import select, create_map_reverse
|
||||
|
||||
|
||||
def read_db_homology(dir_name, filename):
|
||||
df = pd.read_csv(dir_name + "/" + filename, compression='gzip', sep='\t')
|
||||
n = filename.split(".")[0]
|
||||
n = n.split(" ")[0]
|
||||
return df, n
|
||||
|
||||
def read_db_homology(dir_name,filename):
|
||||
df=pd.read_csv(dir_name+"/"+filename,compression='gzip',sep='\t')
|
||||
n=filename.split(".")[0]
|
||||
n=n.split(" ")[0]
|
||||
return df,n
|
||||
|
||||
def get_selection_data():
|
||||
with open("dist_matrix","r") as file:
|
||||
matrix=file.readlines()
|
||||
matrix=[x.split("\t") for x in matrix]
|
||||
matrix=[[float(y) for y in x] for x in matrix]
|
||||
matrix=np.array(matrix)
|
||||
with open("sp_names","r") as file:
|
||||
dname=file.readlines()
|
||||
dname=[x.split("\n")[0] for x in dname]
|
||||
spnmap,nspmap=create_map_reverse(dname)
|
||||
return matrix,spnmap,nspmap
|
||||
with open("dist_matrix", "r") as file:
|
||||
matrix = file.readlines()
|
||||
matrix = [x.split("\t") for x in matrix]
|
||||
matrix = [[float(y) for y in x] for x in matrix]
|
||||
matrix = np.array(matrix)
|
||||
with open("sp_names", "r") as file:
|
||||
dname = file.readlines()
|
||||
dname = [x.split("\n")[0] for x in dname]
|
||||
spnmap, nspmap = create_map_reverse(dname)
|
||||
return matrix, spnmap, nspmap
|
||||
|
||||
def read_select_data(dirname,matrix,spnmap,nspmap,nos):
|
||||
lf=os.listdir(dirname)
|
||||
if len(lf)==0:
|
||||
|
||||
def read_select_data(dirname, matrix, spnmap, nspmap, nos):
|
||||
lf = os.listdir(dirname)
|
||||
if len(lf) == 0:
|
||||
print("No Files in the Directory!!!!!!!")
|
||||
sys.exit(1)
|
||||
if not os.path.isdir("processed"):
|
||||
os.mkdir("processed")
|
||||
for x in lf:
|
||||
df,n=read_db_homology(dirname,x)
|
||||
n=n.split(" ")[0]
|
||||
df=select(df,nos,matrix,spnmap,nspmap,n)
|
||||
(len(df)==nos)
|
||||
indexes=np.array(list(df.index.values))
|
||||
np.save("processed/"+n+"_selected_indexes",indexes)
|
||||
df, n = read_db_homology(dirname, x)
|
||||
n = n.split(" ")[0]
|
||||
df = select(df, nos, matrix, spnmap, nspmap, n)
|
||||
(len(df) == nos)
|
||||
indexes = np.array(list(df.index.values))
|
||||
np.save("processed/" + n + "_selected_indexes", indexes)
|
||||
|
||||
|
||||
def main():
|
||||
arg=sys.argv
|
||||
nos=int(arg[-1])
|
||||
matrix,spnmap,nspmap=get_selection_data()
|
||||
read_select_data("data_homology",matrix,spnmap,nspmap,nos)
|
||||
arg = sys.argv
|
||||
nos = int(arg[-1])
|
||||
matrix, spnmap, nspmap = get_selection_data()
|
||||
read_select_data("data_homology", matrix, spnmap, nspmap, nos)
|
||||
|
||||
if __name__=="__main__":
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
|
|||
157
selector.py
157
selector.py
|
|
@ -1,86 +1,111 @@
|
|||
import pandas as pd
|
||||
import numpy as np
|
||||
|
||||
|
||||
def create_map_reverse(arr):
|
||||
m={}
|
||||
rm={}
|
||||
m = {}
|
||||
rm = {}
|
||||
for i in range(len(arr)):
|
||||
m[arr[i]]=i
|
||||
rm[i]=arr[i]
|
||||
return m,rm
|
||||
|
||||
def get_data_prop(df,nspmap,sp,prop,nos):
|
||||
nos=int(nos*prop)
|
||||
sp=[nspmap[x] for x in sp]
|
||||
data=df[df["homology_species"].isin(sp)]
|
||||
if nos<len(data):
|
||||
random_indexes=np.random.permutation(len(data))
|
||||
data=data.loc[data.index.values[random_indexes[:nos]]]
|
||||
m[arr[i]] = i
|
||||
rm[i] = arr[i]
|
||||
return m, rm
|
||||
|
||||
|
||||
def get_data_prop(df, nspmap, sp, prop, nos):
|
||||
nos = int(nos * prop)
|
||||
sp = [nspmap[x] for x in sp]
|
||||
data = df[df["homology_species"].isin(sp)]
|
||||
if nos < len(data):
|
||||
random_indexes = np.random.permutation(len(data))
|
||||
data = data.loc[data.index.values[random_indexes[:nos]]]
|
||||
return data
|
||||
else:
|
||||
return data
|
||||
|
||||
def create_balanced_dataset_paralog(df,matrix,spnmap,nspmap,nos,hom_type,spname):
|
||||
df=df[df["homology_type"]==hom_type]
|
||||
dist=matrix[spnmap[spname]]
|
||||
dist_sort=np.argsort(dist)
|
||||
sp_far=dist_sort[-5:]
|
||||
sp_near=dist_sort[1:6]
|
||||
#get the records for the species which are far away
|
||||
df_far=get_data_prop(df,nspmap,sp_far,0.2,nos)
|
||||
#get the records for the nearby species
|
||||
df_near=get_data_prop(df,nspmap,sp_near,0.2,nos)
|
||||
df_dist=pd.concat([df_far,df_near])
|
||||
nos=nos-len(df_dist)
|
||||
df=df.drop(df_dist.index.values)
|
||||
random_ind=np.random.permutation(len(df))
|
||||
df_r=df.loc[df.index.values[random_ind[:nos]]]
|
||||
df=pd.concat([df_r,df_dist])
|
||||
|
||||
def create_balanced_dataset_paralog(
|
||||
df,
|
||||
matrix,
|
||||
spnmap,
|
||||
nspmap,
|
||||
nos,
|
||||
hom_type,
|
||||
spname):
|
||||
df = df[df["homology_type"] == hom_type]
|
||||
dist = matrix[spnmap[spname]]
|
||||
dist_sort = np.argsort(dist)
|
||||
sp_far = dist_sort[-5:]
|
||||
sp_near = dist_sort[1:6]
|
||||
# get the records for the species which are far away
|
||||
df_far = get_data_prop(df, nspmap, sp_far, 0.2, nos)
|
||||
# get the records for the nearby species
|
||||
df_near = get_data_prop(df, nspmap, sp_near, 0.2, nos)
|
||||
df_dist = pd.concat([df_far, df_near])
|
||||
nos = nos - len(df_dist)
|
||||
df = df.drop(df_dist.index.values)
|
||||
random_ind = np.random.permutation(len(df))
|
||||
df_r = df.loc[df.index.values[random_ind[:nos]]]
|
||||
df = pd.concat([df_r, df_dist])
|
||||
return df
|
||||
|
||||
def select_data_goc(df,prop,nos):
|
||||
df=df[df["goc_score"]==0.0]
|
||||
nos=int(prop*nos)
|
||||
rind=np.random.permutation(len(df))
|
||||
if len(df)<nos:
|
||||
|
||||
def select_data_goc(df, prop, nos):
|
||||
df = df[df["goc_score"] == 0.0]
|
||||
nos = int(prop * nos)
|
||||
rind = np.random.permutation(len(df))
|
||||
if len(df) < nos:
|
||||
return df
|
||||
else:
|
||||
return df.loc[df.index.values[rind[:nos]]]
|
||||
|
||||
def create_balanced_dataset_ortholog(df,matrix,spnmap,nspmap,nos,hom_type,spname):
|
||||
df=df[df["homology_type"]==hom_type]
|
||||
dist=matrix[spnmap[spname]]
|
||||
dist_sort=np.argsort(dist)
|
||||
sp_far=dist_sort[-5:]
|
||||
sp_near=dist_sort[1:6]
|
||||
#get the records for the species which are far away
|
||||
df_far=get_data_prop(df,nspmap,sp_far,0.2,nos)
|
||||
#get the records for the nearby species
|
||||
df_near=get_data_prop(df,nspmap,sp_near,0.2,nos)
|
||||
df_dist=pd.concat([df_far,df_near])
|
||||
|
||||
|
||||
def create_balanced_dataset_ortholog(
|
||||
df,
|
||||
matrix,
|
||||
spnmap,
|
||||
nspmap,
|
||||
nos,
|
||||
hom_type,
|
||||
spname):
|
||||
df = df[df["homology_type"] == hom_type]
|
||||
dist = matrix[spnmap[spname]]
|
||||
dist_sort = np.argsort(dist)
|
||||
sp_far = dist_sort[-5:]
|
||||
sp_near = dist_sort[1:6]
|
||||
# get the records for the species which are far away
|
||||
df_far = get_data_prop(df, nspmap, sp_far, 0.2, nos)
|
||||
# get the records for the nearby species
|
||||
df_near = get_data_prop(df, nspmap, sp_near, 0.2, nos)
|
||||
df_dist = pd.concat([df_far, df_near])
|
||||
len(df_dist)
|
||||
df=df.drop(df_dist.index.values)
|
||||
df_goc=select_data_goc(df,0.1,nos)
|
||||
df = df.drop(df_dist.index.values)
|
||||
df_goc = select_data_goc(df, 0.1, nos)
|
||||
len(df_goc)
|
||||
df=df.drop(df_goc.index.values)
|
||||
nos=nos-len(df_dist)-len(df_goc)
|
||||
random_ind=np.random.permutation(len(df))
|
||||
df_r=df.loc[df.index.values[random_ind[:nos]]]
|
||||
df=pd.concat([df_r,df_goc,df_dist])
|
||||
df = df.drop(df_goc.index.values)
|
||||
nos = nos - len(df_dist) - len(df_goc)
|
||||
random_ind = np.random.permutation(len(df))
|
||||
df_r = df.loc[df.index.values[random_ind[:nos]]]
|
||||
df = pd.concat([df_r, df_goc, df_dist])
|
||||
return df
|
||||
|
||||
def select(df,nos,matrix,spnmap,nspmap,sp):
|
||||
nos_p=int((0.5*nos)/2)
|
||||
nos_o=int((0.5*nos)/3)
|
||||
|
||||
#get the paralogy data
|
||||
df_p1=create_balanced_dataset_paralog(df,matrix,spnmap,nspmap,nos_p,"within_species_paralog",sp)
|
||||
df_p2=create_balanced_dataset_paralog(df,matrix,spnmap,nspmap,nos_p,"other_paralog",sp)
|
||||
#get the orthology data
|
||||
df_o1=create_balanced_dataset_ortholog(df,matrix,spnmap,nspmap,nos_o,"ortholog_one2many",sp)
|
||||
df_o2=create_balanced_dataset_ortholog(df,matrix,spnmap,nspmap,nos_o,"ortholog_many2many",sp)
|
||||
df_o3=create_balanced_dataset_ortholog(df,matrix,spnmap,nspmap,nos_o,"ortholog_one2one",sp)
|
||||
def select(df, nos, matrix, spnmap, nspmap, sp):
|
||||
nos_p = int((0.5 * nos) / 2)
|
||||
nos_o = int((0.5 * nos) / 3)
|
||||
|
||||
#concatenate everything
|
||||
df=pd.concat([df_o1,df_o2,df_o3,df_p1,df_p2])
|
||||
return df
|
||||
# get the paralogy data
|
||||
df_p1 = create_balanced_dataset_paralog(
|
||||
df, matrix, spnmap, nspmap, nos_p, "within_species_paralog", sp)
|
||||
df_p2 = create_balanced_dataset_paralog(
|
||||
df, matrix, spnmap, nspmap, nos_p, "other_paralog", sp)
|
||||
# get the orthology data
|
||||
df_o1 = create_balanced_dataset_ortholog(
|
||||
df, matrix, spnmap, nspmap, nos_o, "ortholog_one2many", sp)
|
||||
df_o2 = create_balanced_dataset_ortholog(
|
||||
df, matrix, spnmap, nspmap, nos_o, "ortholog_many2many", sp)
|
||||
df_o3 = create_balanced_dataset_ortholog(
|
||||
df, matrix, spnmap, nspmap, nos_o, "ortholog_one2one", sp)
|
||||
|
||||
# concatenate everything
|
||||
df = pd.concat([df_o1, df_o2, df_o3, df_p1, df_p2])
|
||||
return df
|
||||
|
|
|
|||
180
threads.py
180
threads.py
|
|
@ -1,138 +1,144 @@
|
|||
import pandas as pd
|
||||
from threading import Thread
|
||||
from multiprocessing import Process,Lock,Manager
|
||||
from multiprocessing import Process
|
||||
import numpy as np
|
||||
import edlib as ed
|
||||
import pandas as pd
|
||||
import progressbar
|
||||
import time
|
||||
from skbio.alignment import local_pairwise_align_ssw
|
||||
from skbio import DNA,TabularMSA,RNA
|
||||
from skbio import DNA
|
||||
import copy
|
||||
import time
|
||||
from save_data import write_data_synteny
|
||||
|
||||
|
||||
class Thread_objects():
|
||||
def __init__(self,df_temp,gene_sequences,lsy,i,name):
|
||||
self.gene_sequences=copy.deepcopy(gene_sequences)
|
||||
self.lsy=copy.deepcopy(lsy)
|
||||
self.df=copy.deepcopy(df_temp)
|
||||
self.smg=[]
|
||||
self.sml=[]
|
||||
self.indexes=[]
|
||||
self.i=i
|
||||
self.start=0
|
||||
self.end=0
|
||||
self.name=name
|
||||
def __init__(self, df_temp, gene_sequences, lsy, i, name):
|
||||
self.gene_sequences = copy.deepcopy(gene_sequences)
|
||||
self.lsy = copy.deepcopy(lsy)
|
||||
self.df = copy.deepcopy(df_temp)
|
||||
self.smg = []
|
||||
self.sml = []
|
||||
self.indexes = []
|
||||
self.i = i
|
||||
self.start = 0
|
||||
self.end = 0
|
||||
self.name = name
|
||||
|
||||
def create_synteny_matrix_mul(self,gene_seq,g1,g2,n):
|
||||
def create_synteny_matrix_mul(self, gene_seq, g1, g2, n):
|
||||
for gene in g1:
|
||||
if gene=="NULL_GENE":
|
||||
if gene == "NULL_GENE":
|
||||
continue
|
||||
try:
|
||||
temp=gene_seq[gene]
|
||||
except:
|
||||
return np.zeros((n,n,2)),np.zeros((n,n,2))
|
||||
_ = gene_seq[gene]
|
||||
except BaseException:
|
||||
return np.zeros((n, n, 2)), np.zeros((n, n, 2))
|
||||
for gene in g2:
|
||||
if gene=="NULL_GENE":
|
||||
if gene == "NULL_GENE":
|
||||
continue
|
||||
try:
|
||||
temp=gene_seq[gene]
|
||||
except:
|
||||
return np.zeros((n,n,2)),np.zeros((n,n,2))
|
||||
sm=np.zeros((n,n,2))
|
||||
sml=np.zeros((n,n,2))
|
||||
_ = gene_seq[gene]
|
||||
except BaseException:
|
||||
return np.zeros((n, n, 2)), np.zeros((n, n, 2))
|
||||
sm = np.zeros((n, n, 2))
|
||||
sml = np.zeros((n, n, 2))
|
||||
for i in range(n):
|
||||
if g1[i]=="NULL_GENE":
|
||||
if g1[i] == "NULL_GENE":
|
||||
continue
|
||||
if gene_seq[g1[i]]=="":
|
||||
return np.zeros((n,n,2)),np.zeros((n,n,2))
|
||||
if gene_seq[g1[i]] == "":
|
||||
return np.zeros((n, n, 2)), np.zeros((n, n, 2))
|
||||
for j in range(n):
|
||||
if g2[j]=="NULL_GENE":
|
||||
if g2[j] == "NULL_GENE":
|
||||
continue
|
||||
if gene_seq[g2[j]]=="":
|
||||
return np.zeros((n,n,2)),np.zeros((n,n,2))
|
||||
norm_len=max(len(gene_seq[g1[i]]),len(gene_seq[g2[j]]))
|
||||
if gene_seq[g2[j]] == "":
|
||||
return np.zeros((n, n, 2)), np.zeros((n, n, 2))
|
||||
norm_len = max(len(gene_seq[g1[i]]), len(gene_seq[g2[j]]))
|
||||
try:
|
||||
result = ed.align(gene_seq[g1[i]],gene_seq[g2[j]], mode="NW", task="distance")
|
||||
sm[i][j][0]=result["editDistance"]/(norm_len)
|
||||
result = ed.align(gene_seq[g1[i]],gene_seq[g2[j]][::-1], mode="NW", task="distance")
|
||||
sm[i][j][1]=result["editDistance"]/(norm_len)
|
||||
_,result,_=local_pairwise_align_ssw(DNA(gene_seq[g1[i]]),DNA(gene_seq[g2[j]]))
|
||||
sml[i][j][0]=result/(norm_len)
|
||||
_,result,_=local_pairwise_align_ssw(DNA(gene_seq[g1[i]]),DNA(gene_seq[g2[j]][::-1]))
|
||||
sml[i][j][1]=result/(norm_len)
|
||||
except:
|
||||
return np.zeros((n,n,2)),np.zeros((n,n,2))
|
||||
return sm,sml
|
||||
result = ed.align(
|
||||
gene_seq[g1[i]], gene_seq[g2[j]], mode="NW", task="distance")
|
||||
sm[i][j][0] = result["editDistance"] / (norm_len)
|
||||
result = ed.align(
|
||||
gene_seq[g1[i]], gene_seq[g2[j]][::-1], mode="NW", task="distance")
|
||||
sm[i][j][1] = result["editDistance"] / (norm_len)
|
||||
_, result, _ = local_pairwise_align_ssw(
|
||||
DNA(gene_seq[g1[i]]), DNA(gene_seq[g2[j]]))
|
||||
sml[i][j][0] = result / (norm_len)
|
||||
_, result, _ = local_pairwise_align_ssw(
|
||||
DNA(gene_seq[g1[i]]), DNA(gene_seq[g2[j]][::-1]))
|
||||
sml[i][j][1] = result / (norm_len)
|
||||
except BaseException:
|
||||
return np.zeros((n, n, 2)), np.zeros((n, n, 2))
|
||||
return sm, sml
|
||||
|
||||
def synteny_matrix(self,gene_seq,hdf,lsy,n):
|
||||
t=0
|
||||
self.start=time.time()
|
||||
for index,row in progressbar.progressbar(hdf.iterrows()):
|
||||
g1=str(row["gene_stable_id"])
|
||||
g2=str(row["homology_gene_stable_id"])
|
||||
x=[]
|
||||
y=[]
|
||||
t+=1
|
||||
def synteny_matrix(self, gene_seq, hdf, lsy, n):
|
||||
t = 0
|
||||
self.start = time.time()
|
||||
for index, row in progressbar.progressbar(hdf.iterrows()):
|
||||
g1 = str(row["gene_stable_id"])
|
||||
g2 = str(row["homology_gene_stable_id"])
|
||||
x = []
|
||||
y = []
|
||||
t += 1
|
||||
try:
|
||||
temp=lsy[g1]
|
||||
except:
|
||||
_ = lsy[g1]
|
||||
except BaseException:
|
||||
continue
|
||||
try:
|
||||
temp=lsy[g2]
|
||||
except:
|
||||
_ = lsy[g2]
|
||||
except BaseException:
|
||||
continue
|
||||
for i in range(len(lsy[g1]['b'])-1,-1,-1):
|
||||
for i in range(len(lsy[g1]['b']) - 1, -1, -1):
|
||||
x.append(lsy[g1]['b'][i])
|
||||
x.append(g1)
|
||||
for k in lsy[g1]['f']:
|
||||
x.append(k)
|
||||
|
||||
for i in range(len(lsy[g2]['b'])-1,-1,-1):
|
||||
for i in range(len(lsy[g2]['b']) - 1, -1, -1):
|
||||
y.append(lsy[g2]['b'][i])
|
||||
y.append(g2)
|
||||
for k in lsy[g2]['f']:
|
||||
y.append(k)
|
||||
|
||||
assert(len(x)==len(y))
|
||||
assert(len(x)==(2*n+1))
|
||||
smgtemp,smltemp=self.create_synteny_matrix_mul(gene_seq,x,y,2*n+1)
|
||||
if np.all(smgtemp==0):
|
||||
continue
|
||||
|
||||
assert(len(x) == len(y))
|
||||
assert(len(x) == (2 * n + 1))
|
||||
smgtemp, smltemp = self.create_synteny_matrix_mul(
|
||||
gene_seq, x, y, 2 * n + 1)
|
||||
if np.all(smgtemp == 0):
|
||||
continue
|
||||
self.smg.append(smgtemp)
|
||||
self.sml.append(smltemp)
|
||||
self.indexes.append(index)
|
||||
self.end=time.time()
|
||||
print("Thread {} finished in {}s.".format(self.i+1,self.end-self.start))
|
||||
write_data_synteny(self.smg,self.sml,self.indexes,self.i,self.name)
|
||||
self.end = time.time()
|
||||
print(
|
||||
"Thread {} finished in {}s.".format(
|
||||
self.i + 1,
|
||||
self.end - self.start))
|
||||
write_data_synteny(self.smg, self.sml, self.indexes, self.i, self.name)
|
||||
|
||||
|
||||
class Procerssrunner():
|
||||
def __init__(self):
|
||||
self.thread_alive=[]
|
||||
self.obj_list=[]
|
||||
self.thread_alive = []
|
||||
self.obj_list = []
|
||||
|
||||
def start_thread(self,obj,i,thread_alive,n,name):
|
||||
t=Process(target=obj.synteny_matrix,args=(obj.gene_sequences,obj.df,obj.lsy,n),name="Thread_"+str(i+1))
|
||||
print("Thread ",(i+1)," started for ",name,".")
|
||||
def start_thread(self, obj, i, thread_alive, n, name):
|
||||
t = Process(target=obj.synteny_matrix, args=(
|
||||
obj.gene_sequences, obj.df, obj.lsy, n), name="Thread_" + str(i + 1))
|
||||
print("Thread ", (i + 1), " started for ", name, ".")
|
||||
thread_alive.append(t)
|
||||
|
||||
def start_processes(self,nop,df,gene_sequences,lsy,part,n,name):
|
||||
def start_processes(self, nop, df, gene_sequences, lsy, part, n, name):
|
||||
for i in range(nop):
|
||||
df_temp=df.loc[df.index.values[i*part:(i+1)*part]]
|
||||
obj=Thread_objects(df_temp,gene_sequences,lsy,i,name)
|
||||
df_temp = df.loc[df.index.values[i * part:(i + 1) * part]]
|
||||
obj = Thread_objects(df_temp, gene_sequences, lsy, i, name)
|
||||
print("Object Created")
|
||||
self.start_thread(obj,i,self.thread_alive,n,name)
|
||||
self.start_thread(obj, i, self.thread_alive, n, name)
|
||||
self.obj_list.append(obj)
|
||||
st=time.time()
|
||||
st = time.time()
|
||||
for t in self.thread_alive:
|
||||
t.start()
|
||||
#for t in self.thread_alive:
|
||||
#t.join()
|
||||
while(len(self.thread_alive)!=0):
|
||||
# for t in self.thread_alive:
|
||||
# t.join()
|
||||
while(len(self.thread_alive) != 0):
|
||||
time.sleep(60)
|
||||
self.thread_alive=[t for t in self.thread_alive if t.is_alive()]
|
||||
end=time.time()
|
||||
self.thread_alive = [t for t in self.thread_alive if t.is_alive()]
|
||||
end = time.time()
|
||||
print("Ending Processes")
|
||||
print("Time taken:{}s".format(end-st))
|
||||
|
||||
print("Time taken:{}s".format(end - st))
|
||||
|
|
|
|||
61
tree_data.py
61
tree_data.py
|
|
@ -2,45 +2,48 @@ from ete3 import Tree
|
|||
import numpy as np
|
||||
import progressbar
|
||||
|
||||
|
||||
def create_branch_length_padding(bl):
|
||||
maxlen=29
|
||||
maxlen = 29
|
||||
for x in bl:
|
||||
for i in range(len(x),maxlen):
|
||||
for i in range(len(x), maxlen):
|
||||
x.append(0)
|
||||
|
||||
def create_tree_data(treename,df):
|
||||
t=Tree(treename)
|
||||
branch_lengths_s=[]
|
||||
branch_lengths_hs=[]
|
||||
dist=[]
|
||||
ns=[]
|
||||
nhs=[]
|
||||
for index,row in progressbar.progressbar(df.iterrows()):
|
||||
d=0
|
||||
x=row["species"]
|
||||
y=row["homology_species"]
|
||||
bl=[]
|
||||
c=0
|
||||
mca=t.get_common_ancestor(x,y)
|
||||
node=t&x
|
||||
while node.up!=mca:
|
||||
d+=node.dist
|
||||
|
||||
def create_tree_data(treename, df):
|
||||
t = Tree(treename)
|
||||
branch_lengths_s = []
|
||||
branch_lengths_hs = []
|
||||
dist = []
|
||||
ns = []
|
||||
nhs = []
|
||||
for index, row in progressbar.progressbar(df.iterrows()):
|
||||
d = 0
|
||||
x = row["species"]
|
||||
y = row["homology_species"]
|
||||
bl = []
|
||||
c = 0
|
||||
mca = t.get_common_ancestor(x, y)
|
||||
node = t & x
|
||||
while node.up != mca:
|
||||
d += node.dist
|
||||
bl.append(node.dist)
|
||||
node=node.up
|
||||
c+=1
|
||||
node = node.up
|
||||
c += 1
|
||||
ns.append(c)
|
||||
c=0
|
||||
c = 0
|
||||
branch_lengths_s.append(bl)
|
||||
bl=[]
|
||||
node=t&y
|
||||
while node.up!=mca:
|
||||
d+=node.dist
|
||||
bl = []
|
||||
node = t & y
|
||||
while node.up != mca:
|
||||
d += node.dist
|
||||
bl.append(node.dist)
|
||||
node=node.up
|
||||
c+=1
|
||||
node = node.up
|
||||
c += 1
|
||||
nhs.append(c)
|
||||
branch_lengths_hs.append(bl)
|
||||
dist.append(d)
|
||||
create_branch_length_padding(branch_lengths_s)
|
||||
create_branch_length_padding(branch_lengths_hs)
|
||||
return np.array(branch_lengths_s),np.array(branch_lengths_hs),np.array(dist),np.array(ns),np.array(nhs)
|
||||
return np.array(branch_lengths_s), np.array(
|
||||
branch_lengths_hs), np.array(dist), np.array(ns), np.array(nhs)
|
||||
|
|
|
|||
Loading…
Reference in a new issue