mirror of
https://github.com/Priyatham-sai-chand/compara-deep-learning.git
synced 2026-10-05 08:11:34 -07:00
Merge branch 'flake-fix'
This commit is contained in:
commit
7580c07eba
17 changed files with 791 additions and 820 deletions
|
|
@ -3,152 +3,163 @@ import requests
|
|||
import progressbar
|
||||
import sys
|
||||
|
||||
def update_protein(gene_seq,gene):
|
||||
t=0
|
||||
while(t!=2):
|
||||
|
||||
def update_protein(gene_seq, gene):
|
||||
t = 0
|
||||
while(t != 2):
|
||||
try:
|
||||
server = "https://rest.ensembl.org"
|
||||
ext = "/sequence/id/"+str(gene)+"?type=protein;multiple_sequences=1"
|
||||
ext = "/sequence/id/" + \
|
||||
str(gene)+"?type=protein;multiple_sequences=1"
|
||||
|
||||
r = requests.get(server+ext, headers={ "Content-Type" : "application/json"})
|
||||
r = requests.get(
|
||||
server+ext, headers={"Content-Type": "application/json"})
|
||||
|
||||
if not r.ok:
|
||||
r.raise_for_status()
|
||||
sys.exit()
|
||||
r=r.json()
|
||||
if len(r)==1:
|
||||
r=dict(r[0])
|
||||
gene_seq[gene]=str(r["seq"])
|
||||
r = r.json()
|
||||
if len(r) == 1:
|
||||
r = dict(r[0])
|
||||
gene_seq[gene] = str(r["seq"])
|
||||
return
|
||||
else:
|
||||
maxi=0
|
||||
maxlen=0
|
||||
maxi = 0
|
||||
maxlen = 0
|
||||
for i in range(len(r)):
|
||||
m=r[i]
|
||||
m=dict(m)
|
||||
if len(m["seq"])>maxlen:
|
||||
maxi=i
|
||||
r=dict(r[maxi])
|
||||
gene_seq[gene]=str(r["seq"])
|
||||
m = r[i]
|
||||
m = dict(m)
|
||||
if len(m["seq"]) > maxlen:
|
||||
maxi = i
|
||||
r = dict(r[maxi])
|
||||
gene_seq[gene] = str(r["seq"])
|
||||
return
|
||||
except :
|
||||
t+=1
|
||||
#print("\nError:",e)
|
||||
except Exception:
|
||||
t += 1
|
||||
# print("\nError:",e)
|
||||
continue
|
||||
gene_seq[gene]=""
|
||||
gene_seq[gene] = ""
|
||||
|
||||
def update_rest_protein(data,fname):
|
||||
gids={}
|
||||
with open("processed/not_found_"+fname+".json","r") as file:
|
||||
gids=dict(json.load(file))
|
||||
|
||||
gids=list(gids.keys())
|
||||
def update_rest_protein(data, fname):
|
||||
gids = {}
|
||||
with open("processed/not_found_"+fname+".json", "r") as file:
|
||||
gids = dict(json.load(file))
|
||||
|
||||
geneseq={}
|
||||
gids = list(gids.keys())
|
||||
|
||||
geneseq = {}
|
||||
|
||||
server = "https://rest.ensembl.org"
|
||||
ext = "/sequence/id?type=protein"
|
||||
headers={ "Content-Type" : "application/json", "Accept" : "application/json"}
|
||||
headers = {"Content-Type": "application/json",
|
||||
"Accept": "application/json"}
|
||||
|
||||
for i in progressbar.progressbar(range(0,len(gids)-50,50)):
|
||||
ids=dict(ids=list(gids[i:i+50]))
|
||||
for i in progressbar.progressbar(range(0, len(gids)-50, 50)):
|
||||
ids = dict(ids=list(gids[i:i+50]))
|
||||
while(1):
|
||||
try:
|
||||
r = requests.post(server+ext, headers=headers, data=str(json.dumps(ids)))
|
||||
r = requests.post(server+ext, headers=headers,
|
||||
data=str(json.dumps(ids)))
|
||||
if not r.ok:
|
||||
r.raise_for_status()
|
||||
gs=r.json()
|
||||
tgs={}
|
||||
gs = r.json()
|
||||
tgs = {}
|
||||
for g in gs:
|
||||
tgs[g["query"]]=g["seq"]
|
||||
tgs[g["query"]] = g["seq"]
|
||||
geneseq.update(tgs)
|
||||
break
|
||||
except Exception as e:
|
||||
print("Error:",e)
|
||||
print("Error:", e)
|
||||
continue
|
||||
|
||||
data.update(geneseq)
|
||||
for genes in gids:
|
||||
try:
|
||||
_=data[genes]
|
||||
except:
|
||||
_ = data[genes]
|
||||
except Exception:
|
||||
print(genes)
|
||||
update_protein(data,genes)
|
||||
update_protein(data, genes)
|
||||
|
||||
print("Protein Sequences Updated Successfully")
|
||||
return data
|
||||
|
||||
def update(gene_seq,gene):
|
||||
t=0
|
||||
while(t!=2):
|
||||
|
||||
def update(gene_seq, gene):
|
||||
t = 0
|
||||
while(t != 2):
|
||||
try:
|
||||
server = "https://rest.ensembl.org"
|
||||
ext = "/sequence/id/"+str(gene)+"?type=cds;multiple_sequences=1"
|
||||
|
||||
r = requests.get(server+ext, headers={ "Content-Type" : "application/json"})
|
||||
r = requests.get(
|
||||
server+ext, headers={"Content-Type": "application/json"})
|
||||
|
||||
if not r.ok:
|
||||
r.raise_for_status()
|
||||
sys.exit()
|
||||
r=r.json()
|
||||
if len(r)==1:
|
||||
r=dict(r[0])
|
||||
gene_seq[gene]=str(r["seq"])
|
||||
r = r.json()
|
||||
if len(r) == 1:
|
||||
r = dict(r[0])
|
||||
gene_seq[gene] = str(r["seq"])
|
||||
return
|
||||
else:
|
||||
maxi=0
|
||||
maxlen=0
|
||||
maxi = 0
|
||||
maxlen = 0
|
||||
for i in range(len(r)):
|
||||
m=r[i]
|
||||
m=dict(m)
|
||||
if len(m["seq"])>maxlen:
|
||||
maxi=i
|
||||
r=dict(r[maxi])
|
||||
gene_seq[gene]=str(r["seq"])
|
||||
m = r[i]
|
||||
m = dict(m)
|
||||
if len(m["seq"]) > maxlen:
|
||||
maxi = i
|
||||
r = dict(r[maxi])
|
||||
gene_seq[gene] = str(r["seq"])
|
||||
return
|
||||
except Exception as e:
|
||||
t+=1
|
||||
#print("\nError:",e)
|
||||
except Exception:
|
||||
t += 1
|
||||
# print("\nError:",e)
|
||||
continue
|
||||
gene_seq[gene]=""
|
||||
gene_seq[gene] = ""
|
||||
|
||||
def update_rest(data,fname):
|
||||
gids={}
|
||||
with open("processed/not_found_"+fname+".json","r") as file:
|
||||
gids=dict(json.load(file))
|
||||
|
||||
gids=list(gids.keys())
|
||||
def update_rest(data, fname):
|
||||
gids = {}
|
||||
with open("processed/not_found_"+fname+".json", "r") as file:
|
||||
gids = dict(json.load(file))
|
||||
|
||||
geneseq={}
|
||||
gids = list(gids.keys())
|
||||
|
||||
geneseq = {}
|
||||
|
||||
server = "https://rest.ensembl.org"
|
||||
ext = "/sequence/id?type=cds"
|
||||
headers={ "Content-Type" : "application/json", "Accept" : "application/json"}
|
||||
headers = {"Content-Type": "application/json",
|
||||
"Accept": "application/json"}
|
||||
|
||||
for i in progressbar.progressbar(range(0,len(gids)-50,50)):
|
||||
ids=dict(ids=list(gids[i:i+50]))
|
||||
for i in progressbar.progressbar(range(0, len(gids)-50, 50)):
|
||||
ids = dict(ids=list(gids[i:i+50]))
|
||||
while(1):
|
||||
try:
|
||||
r = requests.post(server+ext, headers=headers, data=str(json.dumps(ids)))
|
||||
r = requests.post(server+ext, headers=headers,
|
||||
data=str(json.dumps(ids)))
|
||||
if not r.ok:
|
||||
r.raise_for_status()
|
||||
gs=r.json()
|
||||
tgs={}
|
||||
gs = r.json()
|
||||
tgs = {}
|
||||
for g in gs:
|
||||
tgs[g["query"]]=g["seq"]
|
||||
tgs[g["query"]] = g["seq"]
|
||||
geneseq.update(tgs)
|
||||
break
|
||||
except Exception as e:
|
||||
print("Error:",e)
|
||||
print("Error:", e)
|
||||
continue
|
||||
|
||||
data.update(geneseq)
|
||||
for genes in gids:
|
||||
try:
|
||||
_=data[genes]
|
||||
except:
|
||||
_ = data[genes]
|
||||
except Exception:
|
||||
print(genes)
|
||||
update(data,genes)
|
||||
update(data, genes)
|
||||
|
||||
print("Gene Sequences Updated Successfully")
|
||||
return data
|
||||
|
|
@ -60,7 +60,7 @@ def prepare_features(a_h, d_h, sptree, label):
|
|||
branch_length_species, \
|
||||
branch_length_homology_species, \
|
||||
distance, dist_p_s, dist_p_hs = create_tree_data(
|
||||
sptree, df)
|
||||
sptree, df)
|
||||
assert(len(branch_length_species) == len(df))
|
||||
assert(len(sml) == len(distance))
|
||||
|
||||
|
|
|
|||
69
ftpg.py
69
ftpg.py
|
|
@ -3,30 +3,31 @@ import progressbar
|
|||
import os
|
||||
import sys
|
||||
import urllib.request as urllib
|
||||
import requests
|
||||
|
||||
def download_data(x,dir_name):
|
||||
fname=x.split("/")[-1]
|
||||
path=os.path.join(dir_name,fname)
|
||||
urllib.urlretrieve(x,path)
|
||||
|
||||
def get_data_file(file,dir):
|
||||
def download_data(x, dir_name):
|
||||
fname = x.split("/")[-1]
|
||||
path = os.path.join(dir_name, fname)
|
||||
urllib.urlretrieve(x, path)
|
||||
|
||||
|
||||
def get_data_file(file, dir):
|
||||
if not os.path.isfile(file):
|
||||
print("The specified file does not exist!!!")
|
||||
sys.exit(1)
|
||||
|
||||
with open(file,"r")as f:
|
||||
lf=f.read().splitlines()
|
||||
with open(file, "r")as f:
|
||||
lf = f.read().splitlines()
|
||||
|
||||
if not os.path.exists(dir):
|
||||
os.mkdir(dir)
|
||||
for x in progressbar.progressbar(lf):
|
||||
download_data(x,dir)
|
||||
download_data(x, dir)
|
||||
|
||||
|
||||
|
||||
#This wil download all the fasta files for the coding sequences. To change the directory, change the argument in the get_data_file argument.
|
||||
host ="ftp.ensembl.org"
|
||||
# This wil download all the fasta files for the coding sequences.
|
||||
# To change the directory, change the argument in the get_data_file argument.
|
||||
host = "ftp.ensembl.org"
|
||||
user = "anonymous"
|
||||
password = ""
|
||||
|
||||
|
|
@ -34,50 +35,50 @@ print("Connecting to {}".format(host))
|
|||
ftp = FTP(host)
|
||||
ftp.login(user, password)
|
||||
print("Connected to {}".format(host))
|
||||
base_link="ftp://ftp.ensembl.org"
|
||||
#find sequences of all the cds files
|
||||
l=ftp.nlst("/pub/release-96/fasta")
|
||||
lt=[]
|
||||
for x in l:
|
||||
y=ftp.nlst(x+"/cds")
|
||||
base_link = "ftp://ftp.ensembl.org"
|
||||
# find sequences of all the cds files
|
||||
list_of_files = ftp.nlst("/pub/release-96/fasta")
|
||||
lt = []
|
||||
for x in list_of_files:
|
||||
y = ftp.nlst(x+"/cds")
|
||||
for z in y:
|
||||
if z.endswith(".cds.all.fa.gz"):
|
||||
lt.append(z)
|
||||
|
||||
with open("seq_link.txt","w") as file:
|
||||
with open("seq_link.txt", "w") as file:
|
||||
for x in lt:
|
||||
file.write(base_link+x)
|
||||
file.write("\n")
|
||||
|
||||
#find all the files with protein sequences
|
||||
l=ftp.nlst("/pub/release-96/fasta")
|
||||
lt=[]
|
||||
for x in progressbar.progressbar(l):
|
||||
y=ftp.nlst(x+"/pep")
|
||||
# find all the files with protein sequences
|
||||
list_of_files = ftp.nlst("/pub/release-96/fasta")
|
||||
lt = []
|
||||
for x in progressbar.progressbar(list_of_files):
|
||||
y = ftp.nlst(x+"/pep")
|
||||
for z in y:
|
||||
if z.endswith(".pep.all.fa.gz"):
|
||||
lt.append(z)
|
||||
|
||||
with open("protein_seq.txt","w") as file:
|
||||
with open("protein_seq.txt", "w") as file:
|
||||
for x in lt:
|
||||
file.write(base_link+x)
|
||||
file.write("\n")
|
||||
#get link of all the gtf files
|
||||
l=ftp.nlst("/pub/release-96/gtf")
|
||||
lt=[]
|
||||
for x in l:
|
||||
y=ftp.nlst(x)
|
||||
# get link of all the gtf files
|
||||
list_of_files = ftp.nlst("/pub/release-96/gtf")
|
||||
lt = []
|
||||
for x in list_of_files:
|
||||
y = ftp.nlst(x)
|
||||
for z in y:
|
||||
if z.endswith(".96.gtf.gz"):
|
||||
lt.append(z)
|
||||
|
||||
with open("gtf_link.txt","w") as file:
|
||||
with open("gtf_link.txt", "w") as file:
|
||||
for x in lt:
|
||||
file.write(base_link+x)
|
||||
file.write("\n")
|
||||
|
||||
print("Downloading Data")
|
||||
get_data_file("gtf_link.txt","data")
|
||||
get_data_file("seq_link.txt","geneseq")
|
||||
get_data_file("protein_seq.txt","pro_seq")
|
||||
get_data_file("gtf_link.txt", "data")
|
||||
get_data_file("seq_link.txt", "geneseq")
|
||||
get_data_file("protein_seq.txt", "pro_seq")
|
||||
print("Download Complete.................")
|
||||
|
|
|
|||
219
model.py
219
model.py
|
|
@ -1,132 +1,161 @@
|
|||
dim=10
|
||||
n=3
|
||||
fl=2*n+1
|
||||
maxbl=29
|
||||
|
||||
import tensorflow as tf
|
||||
dim = 10
|
||||
n = 3
|
||||
fl = 2*n+1
|
||||
maxbl = 29
|
||||
|
||||
|
||||
def create_model():
|
||||
tf.reset_default_graph()
|
||||
g=tf.Graph()
|
||||
g = tf.Graph()
|
||||
with g.as_default():
|
||||
synmg=tf.placeholder(dtype=tf.float64,shape=(None,2*n+1,2*n+1,2),name="Synteny_matrix_placeholder_Global")
|
||||
synml=tf.placeholder(dtype=tf.float64,shape=(None,2*n+1,2*n+1,2),name="Synteny_matrix_placeholder_Local")
|
||||
pfam=tf.placeholder(dtype=tf.float64,shape=(None,2*n+1,2*n+1,1),name="Pfam_matrix_placeholder")
|
||||
bls=tf.placeholder(dtype=tf.float64,shape=(None,maxbl),name="Species_Branch_Length_Placeholder")
|
||||
blhs=tf.placeholder(dtype=tf.float64,shape=(None,maxbl),name="Homology_Species_Branch_Length_Placeholder")
|
||||
#gl=tf.placeholder(dtype=tf.float64,shape=(None,1),name="Mean_gene_length")
|
||||
dps=tf.placeholder(dtype=tf.float64,shape=(None,1),name="mca_species_distance")
|
||||
dphs=tf.placeholder(dtype=tf.float64,shape=(None,1),name="mca_homology_species_distance")
|
||||
dis=tf.placeholder(dtype=tf.float64,shape=(None,1),name="total_distance")
|
||||
lr=tf.placeholder(dtype=tf.float64,shape=(),name="learning_rate")
|
||||
y=tf.placeholder(dtype=tf.int32,shape=(None),name="labels")
|
||||
synmg = tf.placeholder(dtype=tf.float64, shape=(
|
||||
None, 2*n+1, 2*n+1, 2), name="Synteny_matrix_placeholder_Global")
|
||||
synml = tf.placeholder(dtype=tf.float64, shape=(
|
||||
None, 2*n+1, 2*n+1, 2), name="Synteny_matrix_placeholder_Local")
|
||||
pfam = tf.placeholder(dtype=tf.float64, shape=(
|
||||
None, 2*n+1, 2*n+1, 1), name="Pfam_matrix_placeholder")
|
||||
bls = tf.placeholder(dtype=tf.float64, shape=(
|
||||
None, maxbl), name="Species_Branch_Length_Placeholder")
|
||||
blhs = tf.placeholder(dtype=tf.float64, shape=(
|
||||
None, maxbl), name="Homology_Species_Branch_Length_Placeholder")
|
||||
# gl=tf.placeholder(dtype=tf.float64,shape=(None,1),name="Mean_gene_length")
|
||||
dps = tf.placeholder(dtype=tf.float64, shape=(
|
||||
None, 1), name="mca_species_distance")
|
||||
dphs = tf.placeholder(dtype=tf.float64, shape=(
|
||||
None, 1), name="mca_homology_species_distance")
|
||||
dis = tf.placeholder(dtype=tf.float64, shape=(
|
||||
None, 1), name="total_distance")
|
||||
lr = tf.placeholder(dtype=tf.float64, shape=(), name="learning_rate")
|
||||
y = tf.placeholder(dtype=tf.int32, shape=(None), name="labels")
|
||||
|
||||
lrs=tf.summary.scalar("Learning_Rate",lr)
|
||||
# lrs = tf.summary.scalar("Learning_Rate", lr)
|
||||
|
||||
x=tf.concat([dps,dps-dphs,dis],1,name="Create_train_vector")
|
||||
x = tf.concat([dps, dps-dphs, dis], 1, name="Create_train_vector")
|
||||
|
||||
print(synmg,"\n",synml,"\n",bls,"\n",blhs,"\n",dps,"\n",dphs,"\n",dis,"\n",x)
|
||||
print(synmg, "\n", synml, "\n", bls, "\n", blhs,
|
||||
"\n", dps, "\n", dphs, "\n", dis, "\n", x)
|
||||
|
||||
reg_l2=tf.contrib.layers.l2_regularizer(0.001)
|
||||
reg_l1 = tf.contrib.layers.l1_regularizer(scale=0.005, scope=None)
|
||||
reg_l2 = tf.contrib.layers.l2_regularizer(0.001)
|
||||
# reg_l1 = tf.contrib.layers.l1_regularizer(scale=0.005, scope=None)
|
||||
|
||||
def get_variable_by_shape(shape,name):
|
||||
f=tf.get_variable(name,shape=shape,initializer=tf.glorot_uniform_initializer(),dtype=tf.float64,regularizer=reg_l2)
|
||||
def get_variable_by_shape(shape, name):
|
||||
f = tf.get_variable(name, shape=shape,
|
||||
initializer=tf.glorot_uniform_initializer(),
|
||||
dtype=tf.float64,
|
||||
regularizer=reg_l2)
|
||||
return f
|
||||
|
||||
def create_synteny_aligner(name,synm):
|
||||
with tf.variable_scope(name+"Synteny_Aligner",reuse=tf.AUTO_REUSE):
|
||||
def create_synteny_aligner(name, synm):
|
||||
with tf.variable_scope(
|
||||
name+"Synteny_Aligner", reuse=tf.AUTO_REUSE):
|
||||
|
||||
fconv=get_variable_by_shape((2,2,2,dim),"fconv")
|
||||
conv=tf.nn.conv2d(synm,fconv,(1,1,1,1),padding="VALID",name="Conv_aligner")
|
||||
fconv = get_variable_by_shape((2, 2, 2, dim), "fconv")
|
||||
conv = tf.nn.conv2d(synm, fconv, (1, 1, 1, 1),
|
||||
padding="VALID", name="Conv_aligner")
|
||||
|
||||
fconv_1=get_variable_by_shape((2,2,dim,dim*2),"fconv_1")
|
||||
conv_1=tf.nn.conv2d(conv,fconv_1,(1,1,1,1),padding="VALID",name="Conv_aligner_1")
|
||||
fconv_1 = get_variable_by_shape((2, 2, dim, dim*2), "fconv_1")
|
||||
conv_1 = tf.nn.conv2d(
|
||||
conv, fconv_1, (1, 1, 1, 1), padding="VALID",
|
||||
name="Conv_aligner_1")
|
||||
|
||||
fxconv=get_variable_by_shape((fl,2,dim*2),"fxconv")
|
||||
x_conv=tf.reshape(synm,(-1,fl*fl,2))
|
||||
x_conv=tf.nn.conv1d(x_conv,fxconv,stride=fl,padding="SAME",name="row_aligner")
|
||||
fxconv = get_variable_by_shape((fl, 2, dim*2), "fxconv")
|
||||
x_conv = tf.reshape(synm, (-1, fl*fl, 2))
|
||||
x_conv = tf.nn.conv1d(
|
||||
x_conv, fxconv, stride=fl, padding="SAME",
|
||||
name="row_aligner")
|
||||
|
||||
y_conv=tf.reshape(tf.transpose(synm,(0,2,1,3)),(-1,fl*fl,2))
|
||||
fyconv=get_variable_by_shape((fl,2,dim*2),"fyconv")
|
||||
y_conv=tf.nn.conv1d(y_conv,fyconv,stride=fl,padding="SAME",name="column_aligner")
|
||||
y_conv = tf.reshape(tf.transpose(
|
||||
synm, (0, 2, 1, 3)), (-1, fl*fl, 2))
|
||||
fyconv = get_variable_by_shape((fl, 2, dim*2), "fyconv")
|
||||
y_conv = tf.nn.conv1d(
|
||||
y_conv, fyconv, stride=fl, padding="SAME",
|
||||
name="column_aligner")
|
||||
|
||||
wconv = get_variable_by_shape((fl, fl, 2, dim*2*10), "wconv")
|
||||
w_conv = tf.nn.conv2d(
|
||||
synm, wconv, (1, 1, 1, 1), padding="VALID",
|
||||
name="Global_Aligner_1")
|
||||
|
||||
wconv=get_variable_by_shape((fl,fl,2,dim*2*10),"wconv")
|
||||
w_conv=tf.nn.conv2d(synm,wconv,(1,1,1,1),padding="VALID",name="Global_Aligner_1")
|
||||
|
||||
conv_1=tf.reshape(conv_1,(-1,25,dim*2))
|
||||
x_conv=tf.reshape(x_conv,(-1,fl,dim*2))
|
||||
y_conv=tf.reshape(y_conv,(-1,fl,dim*2))
|
||||
w_conv=tf.reshape(w_conv,(-1,10,dim*2))
|
||||
conv_final=tf.concat([conv_1,x_conv,y_conv,w_conv],1,name="Concatenate_All_Alignments")
|
||||
conv_1 = tf.reshape(conv_1, (-1, 25, dim*2))
|
||||
x_conv = tf.reshape(x_conv, (-1, fl, dim*2))
|
||||
y_conv = tf.reshape(y_conv, (-1, fl, dim*2))
|
||||
w_conv = tf.reshape(w_conv, (-1, 10, dim*2))
|
||||
conv_final = tf.concat(
|
||||
[conv_1, x_conv, y_conv, w_conv], 1,
|
||||
name="Concatenate_All_Alignments")
|
||||
return conv_final
|
||||
|
||||
with tf.variable_scope("Pfam",reuse=tf.AUTO_REUSE):
|
||||
wconv_pfam=get_variable_by_shape((fl,fl,1,dim*2*10),"wconv_pfam")
|
||||
w_conv_pfam=tf.nn.conv2d(pfam,wconv_pfam,(1,1,1,1),padding="VALID",name="Global_Aligner_pfam")
|
||||
w_conv_pfam=tf.reshape(w_conv_pfam,(-1,10,dim*2))
|
||||
with tf.variable_scope("Pfam", reuse=tf.AUTO_REUSE):
|
||||
wconv_pfam = get_variable_by_shape(
|
||||
(fl, fl, 1, dim*2*10), "wconv_pfam")
|
||||
w_conv_pfam = tf.nn.conv2d(
|
||||
pfam, wconv_pfam, (1, 1, 1, 1), padding="VALID",
|
||||
name="Global_Aligner_pfam")
|
||||
w_conv_pfam = tf.reshape(w_conv_pfam, (-1, 10, dim*2))
|
||||
|
||||
conv_final_g=create_synteny_aligner("Global_",synmg)
|
||||
conv_final_l=create_synteny_aligner("Local_",synml)
|
||||
final=tf.concat([conv_final_g,conv_final_l,w_conv_pfam],1)
|
||||
#final=conv_final_l
|
||||
conv_final_g = create_synteny_aligner("Global_", synmg)
|
||||
conv_final_l = create_synteny_aligner("Local_", synml)
|
||||
final = tf.concat([conv_final_g, conv_final_l, w_conv_pfam], 1)
|
||||
# final=conv_final_l
|
||||
|
||||
with tf.variable_scope("Combine_Renormalize",reuse=tf.AUTO_REUSE):
|
||||
bl=tf.concat([bls,blhs],1)
|
||||
#bl=bls-blhs
|
||||
theta_bl=get_variable_by_shape((maxbl*2,1),"theta_bl")
|
||||
theta_bl=tf.matmul(bl,theta_bl)
|
||||
x=tf.concat([x,theta_bl],1)
|
||||
theta=get_variable_by_shape((4,108),"theta")
|
||||
bias=get_variable_by_shape((1,108),"b")
|
||||
theta_2=tf.matmul(x,theta)+bias
|
||||
theta_2=tf.reshape(theta_2,(-1,108,1))
|
||||
theta_2=tf.tile(theta_2,[1,1,dim*2])
|
||||
final=final*theta_2
|
||||
with tf.variable_scope("Combine_Renormalize", reuse=tf.AUTO_REUSE):
|
||||
bl = tf.concat([bls, blhs], 1)
|
||||
# bl=bls-blhs
|
||||
theta_bl = get_variable_by_shape((maxbl*2, 1), "theta_bl")
|
||||
theta_bl = tf.matmul(bl, theta_bl)
|
||||
x = tf.concat([x, theta_bl], 1)
|
||||
theta = get_variable_by_shape((4, 108), "theta")
|
||||
bias = get_variable_by_shape((1, 108), "b")
|
||||
theta_2 = tf.matmul(x, theta)+bias
|
||||
theta_2 = tf.reshape(theta_2, (-1, 108, 1))
|
||||
theta_2 = tf.tile(theta_2, [1, 1, dim*2])
|
||||
final = final*theta_2
|
||||
print(final)
|
||||
|
||||
flat=tf.layers.flatten(final)
|
||||
flat = tf.layers.flatten(final)
|
||||
|
||||
zero=tf.constant(0.0,dtype=tf.float64)
|
||||
diff=dps-dphs
|
||||
zero = tf.constant(0.0, dtype=tf.float64)
|
||||
diff = dps-dphs
|
||||
print(diff)
|
||||
diff_2=tf.cast(tf.equal(diff,zero),tf.float64)
|
||||
diff_2 = tf.cast(tf.equal(diff, zero), tf.float64)
|
||||
print(diff_2)
|
||||
diff=tf.tile(diff_2,[1,dim*10])
|
||||
diff = tf.tile(diff_2, [1, dim*10])
|
||||
|
||||
flat=tf.concat([flat,diff],1)
|
||||
flat = tf.concat([flat, diff], 1)
|
||||
print(flat)
|
||||
#dense=tf.layers.dense(flat,2048,kernel_regularizer=reg_l2,bias_regularizer=reg_l2)
|
||||
#dense_2=tf.layers.dense(dense,1024,kernel_regularizer=reg_l2,bias_regularizer=reg_l2)
|
||||
dense_3=tf.layers.dense(flat,512,kernel_regularizer=reg_l2,bias_regularizer=reg_l2)
|
||||
# dense=tf.layers.dense(flat,2048,kernel_regularizer=reg_l2,bias_regularizer=reg_l2)
|
||||
# dense_2=tf.layers.dense(dense,1024,kernel_regularizer=reg_l2,bias_regularizer=reg_l2)
|
||||
dense_3 = tf.layers.dense(
|
||||
flat, 512, kernel_regularizer=reg_l2, bias_regularizer=reg_l2)
|
||||
|
||||
logits_pred=tf.layers.dense(dense_3,3,name="Predictions")
|
||||
logits_pred = tf.layers.dense(dense_3, 3, name="Predictions")
|
||||
print(logits_pred)
|
||||
entropy=tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits_pred,labels=y)
|
||||
entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(
|
||||
logits=logits_pred, labels=y)
|
||||
print(entropy)
|
||||
|
||||
#weights = tf.trainable_variables() # all vars of your graph
|
||||
#regl1 = tf.contrib.layers.apply_regularization(reg_l1, weights)
|
||||
# weights = tf.trainable_variables() # all vars of your graph
|
||||
# regl1 = tf.contrib.layers.apply_regularization(reg_l1, weights)
|
||||
reg_losses = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)
|
||||
reg_constant =0.00000001
|
||||
loss=tf.reduce_mean(entropy)+reg_constant * sum(reg_losses)
|
||||
#loss=tf.reduce_mean(entropy)
|
||||
optimizer=tf.train.RMSPropOptimizer(lr)
|
||||
#optimizer=tf.train.AdamOptimizer()
|
||||
losses=tf.summary.scalar("Loss",loss)
|
||||
t_op=optimizer.minimize(loss)
|
||||
acc=tf.math.in_top_k(tf.cast(logits_pred,tf.float32),y,1)
|
||||
accuracy=tf.reduce_mean(tf.cast(acc,tf.float32))
|
||||
accs=tf.summary.scalar("Accuracy",accuracy)
|
||||
summary=tf.summary.merge_all()
|
||||
init=tf.global_variables_initializer()
|
||||
saver=tf.train.Saver()
|
||||
reg_constant = 0.00000001
|
||||
loss = tf.reduce_mean(entropy)+reg_constant * sum(reg_losses)
|
||||
# loss=tf.reduce_mean(entropy)
|
||||
optimizer = tf.train.RMSPropOptimizer(lr)
|
||||
# optimizer=tf.train.AdamOptimizer()
|
||||
# losses = tf.summary.scalar("Loss", loss)
|
||||
t_op = optimizer.minimize(loss)
|
||||
acc = tf.math.in_top_k(tf.cast(logits_pred, tf.float32), y, 1)
|
||||
accuracy = tf.reduce_mean(tf.cast(acc, tf.float32))
|
||||
# accs = tf.summary.scalar("Accuracy", accuracy)
|
||||
summary = tf.summary.merge_all()
|
||||
init = tf.global_variables_initializer()
|
||||
saver = tf.train.Saver()
|
||||
|
||||
for node in (synmg,synml,pfam,bls,blhs,dps,dphs,dis,lr,y):
|
||||
g.add_to_collection("input_nodes",node)
|
||||
for node in (synmg, synml, pfam, bls, blhs, dps, dphs, dis, lr, y):
|
||||
g.add_to_collection("input_nodes", node)
|
||||
|
||||
for node in (loss,t_op,accuracy,init,summary):
|
||||
g.add_to_collection("output_nodes",node)
|
||||
for node in (loss, t_op, accuracy, init, summary):
|
||||
g.add_to_collection("output_nodes", node)
|
||||
|
||||
return g,saver
|
||||
return g, saver
|
||||
|
|
|
|||
|
|
@ -1,62 +1,66 @@
|
|||
import json
|
||||
import gc
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
import pickle
|
||||
import sys
|
||||
import progressbar
|
||||
import os
|
||||
from neighbor_genes import read_genome_maps
|
||||
from process_data import create_data_homology_ls
|
||||
from read_get_gene_seq import read_gene_sequences
|
||||
from access_data_rest import update_rest,update_rest_protein
|
||||
from access_data_rest import update_rest_protein
|
||||
from prepare_synteny_matrix import write_fasta
|
||||
from process_data import create_map_list
|
||||
|
||||
def read_database(fname,dirname):
|
||||
df=pd.read_csv(dirname+"/"+fname,sep="\t",header=None)
|
||||
label_dict=dict(ortholog_one2one=1,
|
||||
other_paralog=0,
|
||||
non_homolog=2,
|
||||
ortholog_one2many=1,
|
||||
ortholog_many2many=1,
|
||||
within_species_paralog=0,
|
||||
gene_split=4)
|
||||
label=[]
|
||||
for _,row in df.iterrows():
|
||||
|
||||
def read_database(fname, dirname):
|
||||
df = pd.read_csv(dirname+"/"+fname, sep="\t", header=None)
|
||||
label_dict = dict(ortholog_one2one=1,
|
||||
other_paralog=0,
|
||||
non_homolog=2,
|
||||
ortholog_one2many=1,
|
||||
ortholog_many2many=1,
|
||||
within_species_paralog=0,
|
||||
gene_split=4)
|
||||
label = []
|
||||
for _, row in df.iterrows():
|
||||
label.append(label_dict[row[7]])
|
||||
df=df.assign(label=label)
|
||||
df=df.drop(7,axis=1)
|
||||
df=df.drop(0,axis=1)
|
||||
df.columns=["gene_stable_id","species","homology_gene_stable_id","homology_species","goc","wga","label"]
|
||||
df = df.assign(label=label)
|
||||
df = df.drop(7, axis=1)
|
||||
df = df.drop(0, axis=1)
|
||||
df.columns = ["gene_stable_id", "species", "homology_gene_stable_id",
|
||||
"homology_species", "goc", "wga", "label"]
|
||||
return df
|
||||
|
||||
|
||||
def read_prediction_file_folder(dir_name):
|
||||
lf=os.listdir(dir_name)
|
||||
a_h=[]
|
||||
d_h=[]
|
||||
lf = os.listdir(dir_name)
|
||||
a_h = []
|
||||
d_h = []
|
||||
for x in progressbar.progressbar(lf):
|
||||
df=read_database(x,dir_name)
|
||||
df = read_database(x, dir_name)
|
||||
a_h.append(df)
|
||||
d_h.append(x.split(".")[0])
|
||||
return a_h,d_h
|
||||
return a_h, d_h
|
||||
|
||||
def create_synteny_features(a_h,d_h,n,a,d,ld,ldg,cmap,cimap,name):
|
||||
lsy=create_data_homology_ls(a_h,d_h,n,a,d,ld,ldg,cmap,cimap,0)
|
||||
protein_sequences=read_gene_sequences(a_h,lsy,"pro_seq","prediction_"+name)
|
||||
protein_sequences=update_rest_protein(protein_sequences,"prediction_"+name)
|
||||
write_fasta(protein_sequences,"prediction_"+name)
|
||||
|
||||
def create_synteny_features(a_h, d_h, n, a, d, ld, ldg, cmap, cimap, name):
|
||||
lsy = create_data_homology_ls(a_h, d_h, n, a, d, ld, ldg, cmap, cimap, 0)
|
||||
protein_sequences = read_gene_sequences(
|
||||
a_h, lsy, "pro_seq", "prediction_"+name)
|
||||
protein_sequences = update_rest_protein(
|
||||
protein_sequences, "prediction_"+name)
|
||||
write_fasta(protein_sequences, "prediction_"+name)
|
||||
print("Protein Sequences Loaded")
|
||||
|
||||
def main():
|
||||
arg=sys.argv
|
||||
dirname=arg[-1]
|
||||
a_h,d_h=read_prediction_file_folder(dirname)
|
||||
n=3
|
||||
|
||||
a,d,ld,ldg,cmap,cimap=read_genome_maps()#read the genome maps
|
||||
def main():
|
||||
arg = sys.argv
|
||||
dirname = arg[-1]
|
||||
a_h, d_h = read_prediction_file_folder(dirname)
|
||||
n = 3
|
||||
|
||||
a, d, ld, ldg, cmap, cimap = read_genome_maps() # read the genome maps
|
||||
print("Genome Maps Loaded.")
|
||||
|
||||
create_synteny_features(a_h,d_h,n,a,d,ld,ldg,cmap,cimap,dirname)
|
||||
if __name__=="__main__":
|
||||
create_synteny_features(a_h, d_h, n, a, d, ld, ldg, cmap, cimap, dirname)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
170
pfam_matrix.py
170
pfam_matrix.py
|
|
@ -4,134 +4,148 @@ import progressbar
|
|||
import os
|
||||
import json
|
||||
import sys
|
||||
from prepare_synteny_matrix import read_data_homology,load_neighbor_genes
|
||||
from prepare_synteny_matrix import read_data_homology, load_neighbor_genes
|
||||
from process_data import create_map_list
|
||||
from process_negative import read_database_txt
|
||||
|
||||
|
||||
def get_score_overlap(x,y,pfam_db,pfam_map):
|
||||
df_1=pfam_db.loc[pfam_map[x]]
|
||||
df_2=pfam_db.loc[pfam_map[y]]
|
||||
l=list(df_2.domain)
|
||||
c=0
|
||||
c_1=0
|
||||
for _,row in df_1.iterrows():
|
||||
if row.domain in l:#check if the domain exists in the list
|
||||
c_1+=1
|
||||
st=int(df_2[df_2["domain"]==row.domain].hmm_from)#get the start
|
||||
end=int(df_2[df_2["domain"]==row.domain].hmm_to)#get the end
|
||||
if (int(row.hmm_from)>st and int(row.hmm_from)<end) or (int(row.hmm_to)>st and int(row.hmm_from)<end):#check if the domain is a ovelapping domain
|
||||
c+=1
|
||||
return c/max(len(df_1),len(df_2))
|
||||
def get_score_overlap(x, y, pfam_db, pfam_map):
|
||||
df_1 = pfam_db.loc[pfam_map[x]]
|
||||
df_2 = pfam_db.loc[pfam_map[y]]
|
||||
list_of_domains = list(df_2.domain)
|
||||
c = 0
|
||||
c_1 = 0
|
||||
for _, row in df_1.iterrows():
|
||||
# check if the domain exists in the list
|
||||
if row.domain in list_of_domains:
|
||||
c_1 += 1
|
||||
# get the start
|
||||
st = int(df_2[df_2["domain"] == row.domain].hmm_from)
|
||||
# get the end
|
||||
end = int(df_2[df_2["domain"] == row.domain].hmm_to)
|
||||
# check if the domain is a ovelapping domain
|
||||
if (int(
|
||||
row.hmm_from) > st and int(row.hmm_from) < end) or (int(
|
||||
row.hmm_to) > st and int(row.hmm_from) < end):
|
||||
c += 1
|
||||
return c/max(len(df_1), len(df_2))
|
||||
|
||||
def pfam_matrix(g1,g2,n,pfam_db,gmap,pfam_map):
|
||||
pm=np.zeros((n,n))
|
||||
|
||||
def pfam_matrix(g1, g2, n, pfam_db, gmap, pfam_map):
|
||||
pm = np.zeros((n, n))
|
||||
for i in range(n):
|
||||
if g1[i]=="NULL_GENE":
|
||||
if g1[i] == "NULL_GENE":
|
||||
continue
|
||||
try:
|
||||
_=gmap[g1[i]]
|
||||
except:
|
||||
_ = gmap[g1[i]]
|
||||
except Exception:
|
||||
continue
|
||||
for j in range(n):
|
||||
if g2[j]=="NULL_GENE":
|
||||
if g2[j] == "NULL_GENE":
|
||||
continue
|
||||
try:
|
||||
_=gmap[g2[j]]
|
||||
except:
|
||||
_ = gmap[g2[j]]
|
||||
except Exception:
|
||||
continue
|
||||
pm[i][j]=get_score_overlap(g1[i],g2[j],pfam_db,pfam_map)
|
||||
pm[i][j] = get_score_overlap(g1[i], g2[j], pfam_db, pfam_map)
|
||||
return pm
|
||||
|
||||
def create_pfam_matrix(df,lsy,pfam_db,pfam_map):
|
||||
n=3
|
||||
glist=list(pfam_db.gene_stable_id)
|
||||
gmap=create_map_list(glist)
|
||||
pg=[]
|
||||
indexes=[]
|
||||
for index,row in progressbar.progressbar(df.iterrows()):
|
||||
g1=str(row["gene_stable_id"])
|
||||
g2=str(row["homology_gene_stable_id"])
|
||||
x=[]
|
||||
y=[]
|
||||
|
||||
def create_pfam_matrix(df, lsy, pfam_db, pfam_map):
|
||||
n = 3
|
||||
glist = list(pfam_db.gene_stable_id)
|
||||
gmap = create_map_list(glist)
|
||||
pg = []
|
||||
indexes = []
|
||||
for index, row in progressbar.progressbar(df.iterrows()):
|
||||
g1 = str(row["gene_stable_id"])
|
||||
g2 = str(row["homology_gene_stable_id"])
|
||||
x = []
|
||||
y = []
|
||||
try:
|
||||
_=lsy[g1]
|
||||
_=lsy[g2]
|
||||
except:
|
||||
_ = lsy[g1]
|
||||
_ = lsy[g2]
|
||||
except Exception:
|
||||
continue
|
||||
for i in range(len(lsy[g1]['b'])-1,-1,-1):
|
||||
for i in range(len(lsy[g1]['b'])-1, -1, -1):
|
||||
x.append(lsy[g1]['b'][i])
|
||||
x.append(g1)
|
||||
for k in lsy[g1]['f']:
|
||||
x.append(k)
|
||||
|
||||
for i in range(len(lsy[g2]['b'])-1,-1,-1):
|
||||
for i in range(len(lsy[g2]['b'])-1, -1, -1):
|
||||
y.append(lsy[g2]['b'][i])
|
||||
y.append(g2)
|
||||
for k in lsy[g2]['f']:
|
||||
y.append(k)
|
||||
|
||||
assert(len(x)==len(y))
|
||||
assert(len(x)==(2*n+1))
|
||||
pmtemp=pfam_matrix(x,y,2*n+1,pfam_db,gmap,pfam_map)
|
||||
assert(len(x) == len(y))
|
||||
assert(len(x) == (2*n+1))
|
||||
pmtemp = pfam_matrix(x, y, 2*n+1, pfam_db, gmap, pfam_map)
|
||||
pg.append(pmtemp)
|
||||
indexes.append(index)
|
||||
return np.array(pg),np.array(indexes)
|
||||
return np.array(pg), np.array(indexes)
|
||||
|
||||
|
||||
def create_pfam_map(pfam_db):
|
||||
pfam_map={}
|
||||
for index,row in progressbar.progressbar(pfam_db.iterrows()):
|
||||
pfam_map = {}
|
||||
for index, row in progressbar.progressbar(pfam_db.iterrows()):
|
||||
try:
|
||||
_=pfam_map[row.gene_stable_id]
|
||||
except:
|
||||
pfam_map[row.gene_stable_id]=[]
|
||||
_ = pfam_map[row.gene_stable_id]
|
||||
except Exception:
|
||||
pfam_map[row.gene_stable_id] = []
|
||||
pfam_map[row.gene_stable_id].append(index)
|
||||
return pfam_map
|
||||
|
||||
|
||||
def main_positive():
|
||||
if not os.path.isdir("processed/pfam_matrices"):
|
||||
os.mkdir("processed/pfam_matrices")
|
||||
a_h,d_h=read_data_homology("data_homology")
|
||||
lsy=load_neighbor_genes()
|
||||
pfam_db=pd.read_hdf("pfam_db_positive.h5")
|
||||
pfam_map=create_pfam_map(pfam_db)
|
||||
ndir="processed/pfam_matrices/"
|
||||
nf1="pfam_matrices"
|
||||
nf3="pfam_indexes"
|
||||
a_h, d_h = read_data_homology("data_homology")
|
||||
lsy = load_neighbor_genes()
|
||||
pfam_db = pd.read_hdf("pfam_db_positive.h5")
|
||||
pfam_map = create_pfam_map(pfam_db)
|
||||
ndir = "processed/pfam_matrices/"
|
||||
nf1 = "pfam_matrices"
|
||||
nf3 = "pfam_indexes"
|
||||
for i in range(len(a_h)):
|
||||
df=a_h[i]
|
||||
df = a_h[i]
|
||||
print(len(df))
|
||||
pfam_matrices,indexes=create_pfam_matrix(df,lsy,pfam_db,pfam_map)
|
||||
np.save(ndir+str(d_h[i])+"_"+nf1,pfam_matrices)
|
||||
np.save(ndir+str(d_h[i])+"_"+nf3,indexes)
|
||||
pfam_matrices, indexes = create_pfam_matrix(df, lsy, pfam_db, pfam_map)
|
||||
np.save(ndir+str(d_h[i])+"_"+nf1, pfam_matrices)
|
||||
np.save(ndir+str(d_h[i])+"_"+nf3, indexes)
|
||||
print(len(indexes))
|
||||
|
||||
|
||||
def read_data_negative(arg):
|
||||
df=read_database_txt(arg[-1])
|
||||
name=arg[-1].split(".")[0]
|
||||
ind=np.load("processed/synteny_matrices/"+name+"_indexes.npy")
|
||||
df=df.loc[ind]
|
||||
pfam_db=pd.read_hdf("pfam_db_negative.h5")
|
||||
pfam_map=create_pfam_map(pfam_db)
|
||||
with open("processed/neighbor_genes_negative.json","r") as file:
|
||||
lsy=dict(json.load(file))
|
||||
return df,pfam_db,pfam_map,lsy,name
|
||||
df = read_database_txt(arg[-1])
|
||||
name = arg[-1].split(".")[0]
|
||||
ind = np.load("processed/synteny_matrices/"+name+"_indexes.npy")
|
||||
df = df.loc[ind]
|
||||
pfam_db = pd.read_hdf("pfam_db_negative.h5")
|
||||
pfam_map = create_pfam_map(pfam_db)
|
||||
with open("processed/neighbor_genes_negative.json", "r") as file:
|
||||
lsy = dict(json.load(file))
|
||||
return df, pfam_db, pfam_map, lsy, name
|
||||
|
||||
|
||||
def main_negative(arg):
|
||||
df,pfam_db,pfam_map,lsy,name=read_data_negative(arg)
|
||||
ndir="processed/pfam_matrices/"
|
||||
nf1="pfam_matrices"
|
||||
nf3="pfam_indexes"
|
||||
df, pfam_db, pfam_map, lsy, name = read_data_negative(arg)
|
||||
ndir = "processed/pfam_matrices/"
|
||||
nf1 = "pfam_matrices"
|
||||
nf3 = "pfam_indexes"
|
||||
print(len(df))
|
||||
pfam_matrices,indexes=create_pfam_matrix(df,lsy,pfam_db,pfam_map)
|
||||
np.save(ndir+name+"_"+nf1,pfam_matrices)
|
||||
np.save(ndir+name+"_"+nf3,indexes)
|
||||
pfam_matrices, indexes = create_pfam_matrix(df, lsy, pfam_db, pfam_map)
|
||||
np.save(ndir+name+"_"+nf1, pfam_matrices)
|
||||
np.save(ndir+name+"_"+nf3, indexes)
|
||||
print(len(indexes))
|
||||
|
||||
|
||||
def main():
|
||||
arg=sys.argv
|
||||
arg = sys.argv
|
||||
main_positive()
|
||||
main_negative(arg)
|
||||
|
||||
if __name__=="__main__":
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
|
|||
|
|
@ -3,48 +3,51 @@ import progressbar
|
|||
import gc
|
||||
import sys
|
||||
|
||||
|
||||
def pfam_parse(filename):
|
||||
rlist=[]
|
||||
rlist = []
|
||||
try:
|
||||
with open(filename) as file:
|
||||
file.seek(0,0)
|
||||
file.seek(0, 0)
|
||||
for line in progressbar.progressbar(file):
|
||||
if line.startswith("#"):
|
||||
continue
|
||||
temp_dict={}
|
||||
x=[y for y in line.split(" ") if y !='']
|
||||
if x[9] !="1":
|
||||
temp_dict = {}
|
||||
x = [y for y in line.split(" ") if y != '']
|
||||
if x[9] != "1":
|
||||
continue
|
||||
temp_dict["gene_stable_id"]=x[3]
|
||||
temp_dict["accession"]=x[1]
|
||||
temp_dict["tlen"]=x[2]
|
||||
temp_dict["qlen"]=x[5]
|
||||
temp_dict["domain"]=x[0]
|
||||
temp_dict["hmm_from"]=x[15]
|
||||
temp_dict["hmm_to"]=x[16]
|
||||
temp_dict["ali_from"]=x[17]
|
||||
temp_dict["ali_to"]=x[18]
|
||||
temp_dict["env_from"]=x[19]
|
||||
temp_dict["env_to"]=x[20]
|
||||
temp_dict["gene_stable_id"] = x[3]
|
||||
temp_dict["accession"] = x[1]
|
||||
temp_dict["tlen"] = x[2]
|
||||
temp_dict["qlen"] = x[5]
|
||||
temp_dict["domain"] = x[0]
|
||||
temp_dict["hmm_from"] = x[15]
|
||||
temp_dict["hmm_to"] = x[16]
|
||||
temp_dict["ali_from"] = x[17]
|
||||
temp_dict["ali_to"] = x[18]
|
||||
temp_dict["env_from"] = x[19]
|
||||
temp_dict["env_to"] = x[20]
|
||||
rlist.append(temp_dict)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
return pd.DataFrame()
|
||||
print(len(rlist))
|
||||
tdf=pd.DataFrame(rlist)
|
||||
tdf = pd.DataFrame(rlist)
|
||||
return tdf
|
||||
|
||||
|
||||
def main():
|
||||
arg=sys.argv
|
||||
fname_1=arg[-2]
|
||||
fname_2=arg[-1]
|
||||
df=pfam_parse(fname_1)
|
||||
df.to_hdf("pfam_db_positive.h5",key="pfam_db_positive",mode="w")
|
||||
df=""
|
||||
arg = sys.argv
|
||||
fname_1 = arg[-2]
|
||||
fname_2 = arg[-1]
|
||||
df = pfam_parse(fname_1)
|
||||
df.to_hdf("pfam_db_positive.h5", key="pfam_db_positive", mode="w")
|
||||
df = ""
|
||||
gc.collect()
|
||||
df=pfam_parse(fname_2)
|
||||
df.to_hdf("pfam_db_negative.h5",key="pfam_db_negative",mode="w")
|
||||
df = pfam_parse(fname_2)
|
||||
df.to_hdf("pfam_db_negative.h5", key="pfam_db_negative", mode="w")
|
||||
print("Pfam Databases Written Successfully :)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
|
|||
|
|
@ -130,7 +130,8 @@ def write_preds(fname, model_name, name, preds, index_dict, df):
|
|||
"_" +
|
||||
name +
|
||||
"_multiple.txt")
|
||||
with open("prediction_" + fname + "_" + model_name + "_" + name + "_multiple.txt", "w") as file:
|
||||
with open("prediction_" + fname + "_" + model_name + "_" + name +
|
||||
"_multiple.txt", "w") as file:
|
||||
for index, row in progressbar.progressbar(df.iterrows()):
|
||||
file.write(str(row[0]))
|
||||
file.write("\t")
|
||||
|
|
|
|||
|
|
@ -1,211 +1,53 @@
|
|||
import json
|
||||
import gc
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
import pickle
|
||||
import tensorflow as tf
|
||||
import sys
|
||||
import os
|
||||
import progressbar
|
||||
from neighbor_genes import read_genome_maps
|
||||
from process_data import create_data_homology_ls
|
||||
from read_get_gene_seq import read_gene_sequences
|
||||
from access_data_rest import update_rest,update_rest_protein
|
||||
from threads import Procerssrunner
|
||||
from prepare_synteny_matrix import read_data_synteny
|
||||
from tree_data import create_tree_data
|
||||
from process_data import create_map_list
|
||||
from pfam_parser import pfam_parse
|
||||
from pfam_matrix import create_pfam_map,create_pfam_matrix
|
||||
import gc
|
||||
import sys
|
||||
|
||||
def read_database(fname):
|
||||
df=pd.read_csv(fname,sep="\t",header=None)
|
||||
label_dict=dict(ortholog_one2one=1,
|
||||
other_paralog=0,
|
||||
non_homolog=2,
|
||||
ortholog_one2many=1,
|
||||
ortholog_many2many=1,
|
||||
within_species_paralog=0,
|
||||
gene_split=4)
|
||||
label=[]
|
||||
for _,row in df.iterrows():
|
||||
label.append(label_dict[row[7]])
|
||||
df=df.assign(label=label)
|
||||
df=df.drop(7,axis=1)
|
||||
df=df.drop(0,axis=1)
|
||||
df.columns=["gene_stable_id","species","homology_gene_stable_id","homology_species","goc","wga","label"]
|
||||
return df
|
||||
|
||||
def select_data_by_length(df,st,end):
|
||||
def pfam_parse(filename):
|
||||
rlist = []
|
||||
try:
|
||||
if end<len(df):
|
||||
if st<end:
|
||||
df=df.loc[df.index.values[st:end]]
|
||||
else:
|
||||
raise ValueError()
|
||||
except:
|
||||
print("Making Predictions for the complete dataframe:)")
|
||||
print(len(df))
|
||||
return df
|
||||
with open(filename) as file:
|
||||
file.seek(0, 0)
|
||||
for line in progressbar.progressbar(file):
|
||||
if line.startswith("#"):
|
||||
continue
|
||||
temp_dict = {}
|
||||
x = [y for y in line.split(" ") if y != '']
|
||||
if x[9] != "1":
|
||||
continue
|
||||
temp_dict["gene_stable_id"] = x[3]
|
||||
temp_dict["accession"] = x[1]
|
||||
temp_dict["tlen"] = x[2]
|
||||
temp_dict["qlen"] = x[5]
|
||||
temp_dict["domain"] = x[0]
|
||||
temp_dict["hmm_from"] = x[15]
|
||||
temp_dict["hmm_to"] = x[16]
|
||||
temp_dict["ali_from"] = x[17]
|
||||
temp_dict["ali_to"] = x[18]
|
||||
temp_dict["env_from"] = x[19]
|
||||
temp_dict["env_to"] = x[20]
|
||||
rlist.append(temp_dict)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
return pd.DataFrame()
|
||||
print(len(rlist))
|
||||
tdf = pd.DataFrame(rlist)
|
||||
return tdf
|
||||
|
||||
def create_synteny_features(a_h,d_h,n,a,d,ld,ldg,cmap,cimap,name):
|
||||
lsy=create_data_homology_ls(a_h,d_h,n,a,d,ld,ldg,cmap,cimap,0)
|
||||
gene_sequences=read_gene_sequences(a_h,lsy,"geneseq","prediction_"+name)
|
||||
gene_sequences=update_rest(gene_sequences,"prediction_"+name)
|
||||
print("Gene Sequences Loaded.")
|
||||
return lsy,gene_sequences
|
||||
|
||||
def threadmaker(nop,df,lsy,gene_sequences,n,name):
|
||||
part=len(df)//nop
|
||||
pr=Procerssrunner()
|
||||
pr.start_processes(nop,df,gene_sequences,lsy,part,n,name)
|
||||
smg,sml,indexes=read_data_synteny(nop,name)
|
||||
sml=np.array(sml)
|
||||
smg=np.array(smg)
|
||||
indexes=np.array(indexes)
|
||||
return sml,smg,indexes
|
||||
|
||||
def get_prediction(smg,sml,pfam_matrices,indexes,bls,blhs,dis,dps,dphs,model_name,no_of_model,w):
|
||||
preds=np.zeros((len(smg),no_of_model))
|
||||
pfam_matrices=pfam_matrices.reshape((len(smg),7,7,1))
|
||||
for i in range(1,no_of_model+1):
|
||||
try:
|
||||
model=tf.train.import_meta_graph(model_name+'_v'+str(i)+'/model.ckpt.meta')
|
||||
except:
|
||||
print("Something wrong with the model.")
|
||||
continue
|
||||
with tf.Session() as sess:
|
||||
try:
|
||||
model.restore(sess,model_name+'_v'+str(i)+"/model.ckpt")
|
||||
graph = tf.get_default_graph()
|
||||
synmgt,synmlt,pfamt,blst,blhst,dpst,dphst,dist,lrt,yt=graph.get_collection("input_nodes")
|
||||
predictions=graph.get_tensor_by_name("Predictions/BiasAdd:0")
|
||||
print("Model Loaded Successfully :)")
|
||||
except:
|
||||
print(":(")
|
||||
sys.exit()
|
||||
|
||||
fd={synmgt:smg,
|
||||
synmlt:sml,
|
||||
pfamt:pfam_matrices,
|
||||
blst:bls,
|
||||
blhst:blhs,
|
||||
dpst:dps.reshape((len(blhs),1)),
|
||||
dist:dis.reshape((len(blhs),1)),
|
||||
dphst:dphs.reshape((len(blhs),1))}
|
||||
preds_t_1=sess.run([predictions],feed_dict=fd)
|
||||
preds_t_1=np.array(preds_t_1)[0]
|
||||
fd={synmgt:smg.transpose((0,2,1,3)),
|
||||
synmlt:sml.transpose((0,2,1,3)),
|
||||
pfamt:pfam_matrices.transpose((0,2,1,3)),
|
||||
blst:blhs,
|
||||
blhst:bls,
|
||||
dpst:dphs.reshape((len(blhs),1)),
|
||||
dist:dis.reshape((len(blhs),1)),
|
||||
dphst:dps.reshape((len(blhs),1))}
|
||||
preds_t_2=sess.run([predictions],feed_dict=fd)
|
||||
preds_t_2=np.array(preds_t_2)[0]
|
||||
preds=preds+w[i-1]*(preds_t_1+preds_t_2)/2
|
||||
tf.reset_default_graph()
|
||||
preds=np.argmax(preds,axis=1)
|
||||
print(preds.shape)
|
||||
return preds
|
||||
|
||||
def write_preds(fname,model_name,name,preds,index_dict,df):
|
||||
print("Writing predcitions to:","prediction_"+fname+"_"+model_name+"_"+name+"_multiple_pfam.txt")
|
||||
with open("prediction_"+fname+"_"+model_name+"_"+name+"_multiple_pfam.txt","w") as file:
|
||||
for index,row in progressbar.progressbar(df.iterrows()):
|
||||
file.write(row["gene_stable_id"])
|
||||
file.write("\t")
|
||||
file.write(row["homology_gene_stable_id"])
|
||||
file.write("\t")
|
||||
file.write(str(row["label"]))
|
||||
file.write("\t")
|
||||
if index in index_dict:
|
||||
file.write(str(preds[index_dict[index]]))
|
||||
file.write("\t")
|
||||
if preds[index_dict[index]]==row["label"]:
|
||||
file.write(str(1))
|
||||
else:
|
||||
file.write(str(0))
|
||||
else:
|
||||
file.write("Error")
|
||||
file.write("\t")
|
||||
file.write("NaN")
|
||||
file.write("\n")
|
||||
|
||||
def main():
|
||||
arg=sys.argv
|
||||
arg=arg[1:]
|
||||
fname=arg[0]
|
||||
model_name=arg[1]
|
||||
no_of_model=int(arg[2])
|
||||
nop=int(arg[3])
|
||||
st=int(arg[4])
|
||||
end=int(arg[5])
|
||||
name=arg[6]
|
||||
pfam_fname=arg[7]
|
||||
weight=arg[8]
|
||||
if weight=="e":
|
||||
w=[1]*no_of_model
|
||||
else:
|
||||
w=[]
|
||||
for i in range(no_of_model):
|
||||
w.append(float(arg[9+i]))
|
||||
df=read_database(fname)
|
||||
df=select_data_by_length(df,st,end)
|
||||
n=3
|
||||
arg = sys.argv
|
||||
fname_1 = arg[-2]
|
||||
fname_2 = arg[-1]
|
||||
df = pfam_parse(fname_1)
|
||||
df.to_hdf("pfam_db_positive.h5", key="pfam_db_positive", mode="w")
|
||||
df = ""
|
||||
gc.collect()
|
||||
df = pfam_parse(fname_2)
|
||||
df.to_hdf("pfam_db_negative.h5", key="pfam_db_negative", mode="w")
|
||||
print("Pfam Databases Written Successfully :)")
|
||||
|
||||
if os.path.exists("prediction_data/data_"+fname):
|
||||
|
||||
with open("prediction_data/data_"+fname,"rb") as file:
|
||||
save_dict=pickle.load(file)
|
||||
smg=save_dict["smg"]
|
||||
sml=save_dict["sml"]
|
||||
pfam_matrices=save_dict["pfam"]
|
||||
indexes=save_dict["indexes"]
|
||||
bls=save_dict["bls"]
|
||||
blhs=save_dict["blhs"]
|
||||
dis=save_dict["dis"]
|
||||
dps=save_dict["dps"]
|
||||
dphs=save_dict["dphs"]
|
||||
|
||||
else:
|
||||
a,d,ld,ldg,cmap,cimap=read_genome_maps()#read the genome maps
|
||||
print("Genome Maps Loaded.")
|
||||
a_h=[df]
|
||||
d_h=["prediction"]
|
||||
|
||||
lsy,gene_sequences=create_synteny_features(a_h,d_h,n,a,d,ld,ldg,cmap,cimap,name)
|
||||
sml,smg,indexes=threadmaker(nop,df,lsy,gene_sequences,n,name)
|
||||
a=""
|
||||
d=""
|
||||
ld=""
|
||||
ldg=""
|
||||
cmap=""
|
||||
cimap=""
|
||||
gene_sequences=""
|
||||
gc.collect()
|
||||
df_temp=df.loc[indexes]
|
||||
pfam_db=pd.read_hdf(pfam_fname.split(".")[0]+"_pfam_db.h5")
|
||||
with open(pfam_fname.split(".")[0]+"_pfam_map","rb") as file:
|
||||
pfam_map=pickle.load(file)
|
||||
pfam_matrices,indexes_pfam=create_pfam_matrix(df_temp,lsy,pfam_db,pfam_map)
|
||||
pfam_db=""
|
||||
gc.collect()
|
||||
bls,blhs,dis,dps,dphs=create_tree_data("species_tree.tree",df_temp)
|
||||
save_dict=dict(smg=smg,sml=sml,pfam=pfam_matrices,indexes=indexes,bls=bls,blhs=blhs,dis=dis,dps=dps,dphs=dphs)
|
||||
|
||||
if not os.path.isdir("prediction_data"):
|
||||
os.mkdir("prediction_data")
|
||||
|
||||
with open("prediction_data/data_"+fname,"wb") as file:
|
||||
pickle.dump(save_dict,file)
|
||||
|
||||
index_dict=create_map_list(indexes)
|
||||
preds=get_prediction(smg,sml,pfam_matrices,indexes,bls,blhs,dis,dps,dphs,model_name,no_of_model,w)
|
||||
|
||||
write_preds(fname,model_name,name,preds,index_dict,df)
|
||||
|
||||
if __name__=="__main__":
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -1,5 +1,4 @@
|
|||
import numpy as np
|
||||
import pandas as pd
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
|
@ -7,95 +6,108 @@ import pickle
|
|||
from select_data import read_db_homology
|
||||
from threads import Procerssrunner
|
||||
from read_get_gene_seq import read_gene_sequences
|
||||
from access_data_rest import update_rest,update_rest_protein
|
||||
from access_data_rest import update_rest, update_rest_protein
|
||||
from Bio import SeqIO
|
||||
from Bio.Seq import Seq
|
||||
from Bio.SeqRecord import SeqRecord
|
||||
from Bio.Alphabet import IUPAC
|
||||
|
||||
def write_fasta(sequences,name):
|
||||
with open(name+".fa","w") as file:
|
||||
for seq in sequences:
|
||||
if sequences[seq]=="":
|
||||
continue
|
||||
record=SeqRecord(Seq(sequences[seq],IUPAC.protein),id=seq)
|
||||
SeqIO.write(record,file,"fasta")
|
||||
|
||||
def read_data_synteny(nop,name):
|
||||
smg=[]
|
||||
sml=[]
|
||||
indexes=[]
|
||||
def write_fasta(sequences, name):
|
||||
with open(name+".fa", "w") as file:
|
||||
for seq in sequences:
|
||||
if sequences[seq] == "":
|
||||
continue
|
||||
record = SeqRecord(Seq(sequences[seq], IUPAC.protein), id=seq)
|
||||
SeqIO.write(record, file, "fasta")
|
||||
|
||||
|
||||
def read_data_synteny(nop, name):
|
||||
smg = []
|
||||
sml = []
|
||||
indexes = []
|
||||
for i in range(nop):
|
||||
try:
|
||||
with open("temp_"+name+"/thread_"+str(i+1)+"_smg.temp","rb") as file:
|
||||
smg=smg+pickle.load(file)
|
||||
with open("temp_"+name+"/thread_"+str(i+1)+"_sml.temp","rb") as file:
|
||||
sml=sml+pickle.load(file)
|
||||
with open("temp_"+name+"/thread_"+str(i+1)+"_indexes.temp","rb") as file:
|
||||
indexes=indexes+pickle.load(file)
|
||||
with open("temp_"+name+"/thread_"+str(i+1) +
|
||||
"_smg.temp", "rb") as file:
|
||||
smg = smg+pickle.load(file)
|
||||
with open("temp_"+name+"/thread_"+str(i+1) +
|
||||
"_sml.temp", "rb") as file:
|
||||
sml = sml+pickle.load(file)
|
||||
with open("temp_"+name+"/thread_"+str(i+1) +
|
||||
"_indexes.temp", "rb") as file:
|
||||
indexes = indexes+pickle.load(file)
|
||||
except Exception as e:
|
||||
print("Problem with thread",i+1,"detected for",name,e)
|
||||
print("Problem with thread", i+1, "detected for", name, e)
|
||||
continue
|
||||
print(len(indexes))
|
||||
return smg,sml,indexes
|
||||
return smg, sml, indexes
|
||||
|
||||
|
||||
def load_neighbor_genes():
|
||||
with open("processed/neighbor_genes.json","r") as file:
|
||||
lsy=dict(json.load(file))
|
||||
with open("processed/neighbor_genes.json", "r") as file:
|
||||
lsy = dict(json.load(file))
|
||||
print(len(lsy))
|
||||
print("Neighbor Genes Loaded")
|
||||
return lsy
|
||||
|
||||
|
||||
def read_data_homology(dirname):
|
||||
lf=os.listdir(dirname)
|
||||
if len(lf)==0:
|
||||
lf = os.listdir(dirname)
|
||||
if len(lf) == 0:
|
||||
print("No Files in the Directory!!!!!!!")
|
||||
sys.exit(1)
|
||||
a_h=[]
|
||||
d_h=[]
|
||||
a_h = []
|
||||
d_h = []
|
||||
for x in lf:
|
||||
df,n=read_db_homology(dirname,x)
|
||||
n=n.split()[0]
|
||||
df, n = read_db_homology(dirname, x)
|
||||
n = n.split()[0]
|
||||
try:
|
||||
indexes=np.load("processed/"+n+"_selected_indexes.npy")
|
||||
except:
|
||||
print("Incomplete data for:",n)
|
||||
df=df.loc[indexes]
|
||||
indexes = np.load("processed/"+n+"_selected_indexes.npy")
|
||||
except Exception:
|
||||
print("Incomplete data for:", n)
|
||||
df = df.loc[indexes]
|
||||
a_h.append(df)
|
||||
d_h.append(n)
|
||||
return a_h,d_h
|
||||
return a_h, d_h
|
||||
|
||||
|
||||
def main():
|
||||
arg=sys.argv
|
||||
nop=int(arg[-1])
|
||||
n=3
|
||||
a_h,d_h=read_data_homology("data_homology")
|
||||
arg = sys.argv
|
||||
nop = int(arg[-1])
|
||||
n = 3
|
||||
a_h, d_h = read_data_homology("data_homology")
|
||||
print("Data Read")
|
||||
lsy=load_neighbor_genes()
|
||||
gene_sequences=read_gene_sequences(a_h,lsy,"geneseq","gene_seq_positive")
|
||||
gene_sequences=update_rest(gene_sequences,"gene_seq_positive")
|
||||
lsy = load_neighbor_genes()
|
||||
gene_sequences = read_gene_sequences(
|
||||
a_h, lsy, "geneseq", "gene_seq_positive")
|
||||
gene_sequences = update_rest(gene_sequences, "gene_seq_positive")
|
||||
print("Gene Sequences Loaded.")
|
||||
if not os.path.isdir("processed/synteny_matrices"):
|
||||
os.mkdir("processed/synteny_matrices")
|
||||
ndir="processed/synteny_matrices/"
|
||||
nf1="synteny_matrices_global"
|
||||
nf2="synteny_matrices_local"
|
||||
nf3="indexes"
|
||||
ndir = "processed/synteny_matrices/"
|
||||
nf1 = "synteny_matrices_global"
|
||||
nf2 = "synteny_matrices_local"
|
||||
nf3 = "indexes"
|
||||
for i in range(len(a_h)):
|
||||
df=a_h[i]
|
||||
part=len(df)//nop
|
||||
pr=Procerssrunner()
|
||||
pr.start_processes(nop,df,gene_sequences,lsy,part,n,d_h[i])
|
||||
smg,sml,indexes=read_data_synteny(nop,d_h[i])
|
||||
df = a_h[i]
|
||||
part = len(df)//nop
|
||||
pr = Procerssrunner()
|
||||
pr.start_processes(nop, df, gene_sequences, lsy, part, n, d_h[i])
|
||||
smg, sml, indexes = read_data_synteny(nop, d_h[i])
|
||||
print(len(indexes))
|
||||
np.save(ndir+str(d_h[i])+"_"+nf1,smg)
|
||||
np.save(ndir+str(d_h[i])+"_"+nf2,sml)
|
||||
np.save(ndir+str(d_h[i])+"_"+nf3,indexes)
|
||||
a_h[i]=df.loc[indexes]
|
||||
np.save(ndir+str(d_h[i])+"_"+nf1, smg)
|
||||
np.save(ndir+str(d_h[i])+"_"+nf2, sml)
|
||||
np.save(ndir+str(d_h[i])+"_"+nf3, indexes)
|
||||
a_h[i] = df.loc[indexes]
|
||||
print("Synteny Matrices Created Successfully :)")
|
||||
protein_sequences=read_gene_sequences(a_h,lsy,"pro_seq","pro_seq_positive")
|
||||
protein_sequences=update_rest_protein(protein_sequences,"pro_seq_positive")
|
||||
write_fasta(protein_sequences,"protein_seq_positive")
|
||||
protein_sequences = read_gene_sequences(
|
||||
a_h, lsy, "pro_seq", "pro_seq_positive")
|
||||
protein_sequences = update_rest_protein(
|
||||
protein_sequences, "pro_seq_positive")
|
||||
write_fasta(protein_sequences, "protein_seq_positive")
|
||||
print("Protein Sequences Loaded.")
|
||||
if __name__=="__main__":
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
|
|||
|
|
@ -3,10 +3,10 @@ import progressbar
|
|||
from save_data import write_dict_json
|
||||
|
||||
|
||||
def create_map_list(l): # this function maps the indexes to values
|
||||
def create_map_list(lister): # this function maps the indexes to values
|
||||
t = {}
|
||||
for i in range(len(l)):
|
||||
t[l[i]] = i
|
||||
for i in range(len(lister)):
|
||||
t[lister[i]] = i
|
||||
return t
|
||||
|
||||
|
||||
|
|
@ -58,10 +58,15 @@ def get_nearest_neighbors(g, gs, n, a, d, ld, ldg, cmap, cimap):
|
|||
try:
|
||||
_ = ld[gi] # see if the corresponding gene map exists
|
||||
except BaseException:
|
||||
# print("Length of Dataframes:{} \t Length of Loaded Genes:{} \t Length of Loaded Genomes Dictionaries:{}".format(len(a),len(ld),len(ldg)))
|
||||
# print("Length of Dataframes:{}
|
||||
# Length of Loaded Genes:{}
|
||||
# Length of Loaded Genomes Dictionaries:
|
||||
# {}".format(len(a),len(ld),len(ldg)))
|
||||
return ne, nr
|
||||
sldg = ldg[gi] # select the corresponding map
|
||||
if g not in sldg: # if the gene is not present in the dataframe return empty lists
|
||||
# if the gene is not present in the dataframe
|
||||
# return empty lists
|
||||
if g not in sldg:
|
||||
# print(g,"\t",gs)
|
||||
return ne, nr
|
||||
i = sldg[g] # find the index of the gene
|
||||
|
|
@ -80,9 +85,11 @@ def get_nearest_neighbors(g, gs, n, a, d, ld, ldg, cmap, cimap):
|
|||
end = list(sldf.end)
|
||||
end = np.array(end)
|
||||
assert(len(end) == len(sldf))
|
||||
end = end - start # subtract start from it so as to get relative position
|
||||
# subtract start from it so as to get relative position
|
||||
end = end - start
|
||||
end_s = np.argsort(end) # sort them by the order of distance
|
||||
if end[end_s[0]] >= 0: # if all the genes end ahead of the one in consideration
|
||||
# if all the genes end ahead of the one in consideration
|
||||
if end[end_s[0]] >= 0:
|
||||
flag = 1 # increment the pointer
|
||||
ne.append("NULL_GENE") # append the NULL_GENE value
|
||||
continue
|
||||
|
|
@ -144,7 +151,8 @@ def create_data_homology_ls(a_h, d_h, n, a, d, ld, ldg, cmap, cimap, to_write):
|
|||
xl, xr = get_nearest_neighbors(
|
||||
x, xs, n, a, d, ld, ldg, cmap, cimap)
|
||||
if len(
|
||||
xl) != 0: # check if neighboring genes were successfully found
|
||||
# check if neighboring genes were successfully found
|
||||
xl) != 0:
|
||||
lsy[x] = dict(b=xl, f=xr)
|
||||
lsytemp[x] = dict(b=xl, f=xr)
|
||||
except BaseException:
|
||||
|
|
|
|||
|
|
@ -1,63 +1,63 @@
|
|||
import pandas as pd
|
||||
import numpy as np
|
||||
import os
|
||||
import sys
|
||||
import progressbar
|
||||
import json
|
||||
import sys
|
||||
from neighbor_genes import read_genome_maps
|
||||
from process_data import create_data_homology_ls
|
||||
from threads import Procerssrunner
|
||||
from read_get_gene_seq import read_gene_sequences
|
||||
from access_data_rest import update_rest,update_rest_protein
|
||||
from prepare_synteny_matrix import read_data_synteny,write_fasta
|
||||
from access_data_rest import update_rest, update_rest_protein
|
||||
from prepare_synteny_matrix import read_data_synteny, write_fasta
|
||||
from save_data import write_dict_json
|
||||
from access_data_rest import update_rest_protein
|
||||
|
||||
|
||||
def read_database_txt(filename):
|
||||
df=pd.read_csv(filename,sep="\t",header=None)
|
||||
df=df.drop(0,axis=1)
|
||||
df.columns=["gene_stable_id","species","homology_gene_stable_id","homology_species","wga","goc","homology_type"]
|
||||
df = pd.read_csv(filename, sep="\t", header=None)
|
||||
df = df.drop(0, axis=1)
|
||||
df.columns = ["gene_stable_id", "species", "homology_gene_stable_id",
|
||||
"homology_species", "wga", "goc", "homology_type"]
|
||||
return df
|
||||
|
||||
|
||||
def main():
|
||||
arg=sys.argv
|
||||
a,d,ld,ldg,cmap,cimap=read_genome_maps()
|
||||
arg = sys.argv
|
||||
a, d, ld, ldg, cmap, cimap = read_genome_maps()
|
||||
print("Genome Maps Loaded.")
|
||||
df=read_database_txt(arg[-2])
|
||||
nop=int(arg[-1])
|
||||
df = read_database_txt(arg[-2])
|
||||
nop = int(arg[-1])
|
||||
print("Data Read.")
|
||||
a_h=[]
|
||||
d_h=[]
|
||||
a_h = []
|
||||
d_h = []
|
||||
a_h.append(df)
|
||||
d_h.append(arg[-2].split(".")[0])
|
||||
n=3
|
||||
lsy=create_data_homology_ls(a_h,d_h,n,a,d,ld,ldg,cmap,cimap,0)
|
||||
write_dict_json("neighbor_genes_negative","processed",lsy)
|
||||
n = 3
|
||||
lsy = create_data_homology_ls(a_h, d_h, n, a, d, ld, ldg, cmap, cimap, 0)
|
||||
write_dict_json("neighbor_genes_negative", "processed", lsy)
|
||||
print("Neighbor Genes Found and Saved Successfully:)")
|
||||
gene_sequences=read_gene_sequences(a_h,lsy,"geneseq","gene_seq_negative")
|
||||
gene_sequences=update_rest(gene_sequences,"gene_seq_negative")
|
||||
ndir="processed/synteny_matrices/"
|
||||
nf1="synteny_matrices_global"
|
||||
nf2="synteny_matrices_local"
|
||||
nf3="indexes"
|
||||
gene_sequences = read_gene_sequences(
|
||||
a_h, lsy, "geneseq", "gene_seq_negative")
|
||||
gene_sequences = update_rest(gene_sequences, "gene_seq_negative")
|
||||
ndir = "processed/synteny_matrices/"
|
||||
nf1 = "synteny_matrices_global"
|
||||
nf2 = "synteny_matrices_local"
|
||||
nf3 = "indexes"
|
||||
for i in range(len(a_h)):
|
||||
df=a_h[i]
|
||||
part=len(df)//nop
|
||||
pr=Procerssrunner()
|
||||
pr.start_processes(nop,df,gene_sequences,lsy,part,n,d_h[i])
|
||||
smg,sml,indexes=read_data_synteny(nop,d_h[i])
|
||||
df = a_h[i]
|
||||
part = len(df)//nop
|
||||
pr = Procerssrunner()
|
||||
pr.start_processes(nop, df, gene_sequences, lsy, part, n, d_h[i])
|
||||
smg, sml, indexes = read_data_synteny(nop, d_h[i])
|
||||
print(len(indexes))
|
||||
np.save(ndir+str(d_h[i])+"_"+nf1,smg)
|
||||
np.save(ndir+str(d_h[i])+"_"+nf2,sml)
|
||||
np.save(ndir+str(d_h[i])+"_"+nf3,indexes)
|
||||
a_h[i]=df.loc[indexes]
|
||||
np.save(ndir+str(d_h[i])+"_"+nf1, smg)
|
||||
np.save(ndir+str(d_h[i])+"_"+nf2, sml)
|
||||
np.save(ndir+str(d_h[i])+"_"+nf3, indexes)
|
||||
a_h[i] = df.loc[indexes]
|
||||
print("Synteny Matrices Created Successfully :)")
|
||||
protein_sequences=read_gene_sequences(a_h,lsy,"pro_seq","pro_seq_negative")
|
||||
protein_sequences=update_rest_protein(protein_sequences,"pro_seq_negative")
|
||||
write_fasta(protein_sequences,"pro_seq_negative")
|
||||
protein_sequences = read_gene_sequences(
|
||||
a_h, lsy, "pro_seq", "pro_seq_negative")
|
||||
protein_sequences = update_rest_protein(
|
||||
protein_sequences, "pro_seq_negative")
|
||||
write_fasta(protein_sequences, "pro_seq_negative")
|
||||
|
||||
if __name__=="__main__":
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
|
|
|
|||
|
|
@ -63,7 +63,8 @@ def read_data_genome(dir_name, a, dict_ind_genome):
|
|||
print(e)
|
||||
continue
|
||||
# print(data_gene[0:10])
|
||||
data_gene = data_gene[(data_gene['gene_biotype'] == 'protein_coding') | (
|
||||
data_gene = data_gene[(
|
||||
data_gene['gene_biotype'] == 'protein_coding') | (
|
||||
data_gene['gene_source'] == 'protein_coding')]
|
||||
# print(data_gene[data_gene["gene_id"]=="ENSNGAG00000000407"])
|
||||
a.append(data_gene)
|
||||
|
|
|
|||
|
|
@ -27,7 +27,8 @@ def create_dict(keys, values, dictionary):
|
|||
return dictionary
|
||||
|
||||
# this function maps all the genes to their respective species.
|
||||
# (Function: when finding the species of any gene we do not need to search the entire dataframe)
|
||||
# (Function: when finding the species of any gene
|
||||
# we do not need to search the entire dataframe)
|
||||
|
||||
|
||||
def group_seq_by_species(df, g_to_sp):
|
||||
|
|
@ -64,7 +65,9 @@ def read_gene_seq(dirname, s, genes_by_species):
|
|||
ftr = []
|
||||
for f in lof:
|
||||
if f.split(".")[
|
||||
0] in s: # check whether the species is present in the species to read list. Will skip those species which are not present in the dataframe
|
||||
# check whether the species is present in the species to read list.
|
||||
# Will skip those species which are not present in the dataframe
|
||||
0] in s:
|
||||
ftr.append(f)
|
||||
data = {}
|
||||
for f in progressbar.progressbar(ftr):
|
||||
|
|
@ -73,17 +76,18 @@ def read_gene_seq(dirname, s, genes_by_species):
|
|||
record = SeqIO.parse(file, "fasta")
|
||||
for r in record:
|
||||
gid, gbt = description_cleaner(r.description)
|
||||
if str(gid) not in data and str(
|
||||
gid) in genes_by_species[species] and gbt == "protein_coding":
|
||||
if str(gid) not in data and str(gid) in \
|
||||
genes_by_species[species] and gbt == "protein_coding":
|
||||
data[gid] = str(r.seq)
|
||||
return data
|
||||
|
||||
|
||||
def read_gene_sequences(hdf, lsy, data_dir, fname):
|
||||
"""The basic idea here is to create a list/dictionary of all the genes by their species.
|
||||
Once the mapping is done, all the respective fasta sequence files are read by Species
|
||||
and the CDNA sequences for each gene in the species record are read and stored.
|
||||
Thus we don't have to read the same file multiple times."""
|
||||
"""The basic idea here is to create a list/dictionary of all the genes by
|
||||
their species. Once the mapping is done, all the respective fasta sequence
|
||||
files are read by Species and the CDNA sequences for each gene in the
|
||||
species record are read and stored.
|
||||
Thus we don't have to read the same file multiple times."""
|
||||
grouped_genes = {}
|
||||
gene_by_species_dict = {}
|
||||
for df in progressbar.progressbar(hdf):
|
||||
|
|
|
|||
|
|
@ -15,9 +15,12 @@ def write_dict_json(name, dir, d):
|
|||
def write_data_synteny(smg, sml, indexes, i, name):
|
||||
if not os.path.exists("temp_" + name):
|
||||
os.mkdir("temp_" + name)
|
||||
with open("temp_" + name + "/thread_" + str(i + 1) + "_smg.temp", "wb") as file:
|
||||
with open("temp_" + name + "/thread_" + str(i + 1) +
|
||||
"_smg.temp", "wb") as file:
|
||||
pickle.dump(smg, file)
|
||||
with open("temp_" + name + "/thread_" + str(i + 1) + "_sml.temp", "wb") as file:
|
||||
with open("temp_" + name + "/thread_" + str(i + 1) +
|
||||
"_sml.temp", "wb") as file:
|
||||
pickle.dump(sml, file)
|
||||
with open("temp_" + name + "/thread_" + str(i + 1) + "_indexes.temp", "wb") as file:
|
||||
with open("temp_" + name + "/thread_" + str(i + 1) +
|
||||
"_indexes.temp", "wb") as file:
|
||||
pickle.dump(indexes, file)
|
||||
|
|
|
|||
|
|
@ -52,10 +52,12 @@ class Thread_objects():
|
|||
norm_len = max(len(gene_seq[g1[i]]), len(gene_seq[g2[j]]))
|
||||
try:
|
||||
result = ed.align(
|
||||
gene_seq[g1[i]], gene_seq[g2[j]], mode="NW", task="distance")
|
||||
gene_seq[g1[i]], gene_seq[g2[j]],
|
||||
mode="NW", task="distance")
|
||||
sm[i][j][0] = result["editDistance"] / (norm_len)
|
||||
result = ed.align(
|
||||
gene_seq[g1[i]], gene_seq[g2[j]][::-1], mode="NW", task="distance")
|
||||
gene_seq[g1[i]], gene_seq[g2[j]][::-1], mode="NW",
|
||||
task="distance")
|
||||
sm[i][j][1] = result["editDistance"] / (norm_len)
|
||||
_, result, _ = local_pairwise_align_ssw(
|
||||
DNA(gene_seq[g1[i]]), DNA(gene_seq[g2[j]]))
|
||||
|
|
@ -120,7 +122,8 @@ class Procerssrunner():
|
|||
|
||||
def start_thread(self, obj, i, thread_alive, n, name):
|
||||
t = Process(target=obj.synteny_matrix, args=(
|
||||
obj.gene_sequences, obj.df, obj.lsy, n), name="Thread_" + str(i + 1))
|
||||
obj.gene_sequences, obj.df, obj.lsy, n),
|
||||
name="Thread_" + str(i + 1))
|
||||
print("Thread ", (i + 1), " started for ", name, ".")
|
||||
thread_alive.append(t)
|
||||
|
||||
|
|
|
|||
283
train.py
283
train.py
|
|
@ -4,87 +4,119 @@ import sys
|
|||
import tensorflow as tf
|
||||
from model import create_model
|
||||
|
||||
|
||||
def create_branch_length_padding(bl):
|
||||
maxlen=0
|
||||
maxlen = 0
|
||||
for x in bl:
|
||||
if len(x)>maxlen:
|
||||
maxlen=len(x)
|
||||
if len(x) > maxlen:
|
||||
maxlen = len(x)
|
||||
|
||||
for x in range(len(bl)):
|
||||
temp=bl[x]
|
||||
for i in range(len(temp),maxlen):
|
||||
temp=np.append(temp,[0])
|
||||
bl[x]=temp
|
||||
temp = bl[x]
|
||||
for i in range(len(temp), maxlen):
|
||||
temp = np.append(temp, [0])
|
||||
bl[x] = temp
|
||||
return bl
|
||||
|
||||
def train(train_synteny_matrices_global,train_synteny_matrices_local,train_pfam_matrices,train_branch_length_species,train_branch_length_homology_species,train_dist_p_s,train_dist_p_hs,train_distance,train_labels,v,num_epochs,learning_rate,decay,size_train,batch_size,model_name):
|
||||
print("Going to train model {} for:\n Batch Size:{} \n Learning Rate:{} \n Decay:{}\n On {} Samples".format(v,batch_size,learning_rate,decay,len(train_synteny_matrices_global)))
|
||||
graph,saver=create_model()
|
||||
synmg,synml,pfam,bls,blhs,dps,dphs,dis,lr,y=graph.get_collection("input_nodes")
|
||||
loss,t_op,accuracy,init,summary=graph.get_collection("output_nodes")
|
||||
|
||||
def train(train_synteny_matrices_global, train_synteny_matrices_local,
|
||||
train_pfam_matrices, train_branch_length_species,
|
||||
train_branch_length_homology_species, train_dist_p_s,
|
||||
train_dist_p_hs, train_distance, train_labels, v,
|
||||
num_epochs, learning_rate, decay, size_train,
|
||||
batch_size, model_name):
|
||||
print("Going to train model {} for:\n Batch Size:{} \
|
||||
\n Learning Rate:{} \n Decay:{}\n On {} Samples".format(
|
||||
v, batch_size, learning_rate, decay,
|
||||
len(train_synteny_matrices_global)
|
||||
))
|
||||
graph, saver = create_model()
|
||||
synmg, synml, pfam, bls, blhs, dps, \
|
||||
dphs, dis, lr, y = graph.get_collection("input_nodes")
|
||||
loss, t_op, accuracy, init, summary = graph.get_collection("output_nodes")
|
||||
with tf.Session(graph=graph) as sess:
|
||||
writer = tf.summary.FileWriter('./'+model_name+'_v'+str(v), sess.graph)
|
||||
sess.run(init)
|
||||
learn=learning_rate
|
||||
learn = learning_rate
|
||||
for j in range(num_epochs):
|
||||
for i in range(size_train//batch_size):
|
||||
feed_dict={
|
||||
synmg:train_synteny_matrices_global[i*batch_size:(i+1)*batch_size],
|
||||
synml:train_synteny_matrices_local[i*batch_size:(i+1)*batch_size],
|
||||
pfam:train_pfam_matrices[i*batch_size:(i+1)*batch_size].reshape((batch_size,7,7,1)),
|
||||
bls:train_branch_length_species[i*batch_size:(i+1)*batch_size],
|
||||
blhs:train_branch_length_homology_species[i*batch_size:(i+1)*batch_size],
|
||||
dps:train_dist_p_s[i*batch_size:(i+1)*batch_size].reshape((batch_size,1)),
|
||||
dphs:train_dist_p_hs[i*batch_size:(i+1)*batch_size].reshape((batch_size,1)),
|
||||
dis:train_distance[i*batch_size:(i+1)*batch_size].reshape((batch_size,1)),
|
||||
lr:learn,
|
||||
y:train_labels[i*batch_size:(i+1)*batch_size]
|
||||
feed_dict = {
|
||||
synmg: train_synteny_matrices_global
|
||||
[i*batch_size:(i+1)*batch_size],
|
||||
synml: train_synteny_matrices_local
|
||||
[i*batch_size:(i+1)*batch_size],
|
||||
pfam: train_pfam_matrices
|
||||
[i*batch_size:(i+1)*batch_size]
|
||||
.reshape((batch_size, 7, 7, 1)),
|
||||
bls: train_branch_length_species
|
||||
[i*batch_size:(i+1)*batch_size],
|
||||
blhs: train_branch_length_homology_species
|
||||
[i*batch_size:(i+1)*batch_size],
|
||||
dps: train_dist_p_s
|
||||
[i*batch_size:(i+1)*batch_size]
|
||||
.reshape((batch_size, 1)),
|
||||
dphs: train_dist_p_hs
|
||||
[i*batch_size:(i+1)*batch_size]
|
||||
.reshape((batch_size, 1)),
|
||||
dis: train_distance
|
||||
[i*batch_size:(i+1)*batch_size]
|
||||
.reshape((batch_size, 1)),
|
||||
lr: learn,
|
||||
y: train_labels[i*batch_size:(i+1)*batch_size]
|
||||
}
|
||||
sess.run(t_op,feed_dict=feed_dict)
|
||||
sess.run(t_op, feed_dict=feed_dict)
|
||||
|
||||
test_dict={
|
||||
synmg:train_synteny_matrices_global[size_train:],
|
||||
synml:train_synteny_matrices_local[size_train:],
|
||||
pfam:train_pfam_matrices[size_train:].reshape((len(train_synteny_matrices_global)-size_train,7,7,1)),
|
||||
bls:train_branch_length_species[size_train:],
|
||||
blhs:train_branch_length_homology_species[size_train:],
|
||||
dps:train_dist_p_s[size_train:].reshape((len(train_synteny_matrices_global)-size_train,1)),
|
||||
dphs:train_dist_p_hs[size_train:].reshape((len(train_synteny_matrices_global)-size_train,1)),
|
||||
dis:train_distance[size_train:].reshape((len(train_synteny_matrices_global)-size_train,1)),
|
||||
y:train_labels[size_train:],
|
||||
lr:learn
|
||||
}
|
||||
accuracy_test,loss_test,summary_write=sess.run([accuracy,loss,summary],feed_dict=test_dict)
|
||||
writer.add_summary(summary_write,i+1)
|
||||
print("Epoch:{} Test Accuracy:{} Test Loss:{}".format(j+1,accuracy_test*100,loss_test))
|
||||
learn*=decay
|
||||
saver.save(sess,model_name+"_v"+str(v)+"/model.ckpt")
|
||||
test_dict = {
|
||||
synmg: train_synteny_matrices_global[size_train:],
|
||||
synml: train_synteny_matrices_local[size_train:],
|
||||
pfam: train_pfam_matrices[size_train:]
|
||||
.reshape((
|
||||
len(train_synteny_matrices_global)-size_train, 7, 7, 1)),
|
||||
bls: train_branch_length_species[size_train:],
|
||||
blhs: train_branch_length_homology_species[size_train:],
|
||||
dps: train_dist_p_s[size_train:]
|
||||
.reshape((len(train_synteny_matrices_global)-size_train, 1)),
|
||||
dphs: train_dist_p_hs[size_train:]
|
||||
.reshape((len(train_synteny_matrices_global)-size_train, 1)),
|
||||
dis: train_distance[size_train:]
|
||||
.reshape((len(train_synteny_matrices_global)-size_train, 1)),
|
||||
y: train_labels[size_train:],
|
||||
lr: learn
|
||||
}
|
||||
accuracy_test, loss_test, summary_write = sess.run(
|
||||
[accuracy, loss, summary], feed_dict=test_dict)
|
||||
writer.add_summary(summary_write, i+1)
|
||||
print("Epoch:{} Test Accuracy:{} Test Loss:{}".format(
|
||||
j+1, accuracy_test*100, loss_test))
|
||||
learn *= decay
|
||||
saver.save(sess, model_name+"_v"+str(v)+"/model.ckpt")
|
||||
writer.close()
|
||||
|
||||
def read_positive(len_p,bls,blhs,dis,dps,dphs,sml,smg,pfam,label):
|
||||
rowsh=[]
|
||||
with open("dataset","rb") as file:
|
||||
rowsh=pickle.load(file)
|
||||
shi=np.random.permutation(len(rowsh))
|
||||
rows_shuffled=[]
|
||||
|
||||
def read_positive(len_p, bls, blhs, dis, dps, dphs, sml, smg, pfam, label):
|
||||
rowsh = []
|
||||
with open("dataset", "rb") as file:
|
||||
rowsh = pickle.load(file)
|
||||
shi = np.random.permutation(len(rowsh))
|
||||
rows_shuffled = []
|
||||
for i in range(len(shi)):
|
||||
rows_shuffled.append(rowsh[shi[i]])
|
||||
rowsh=rows_shuffled
|
||||
spco={}
|
||||
spcp={}
|
||||
rowsh = rows_shuffled
|
||||
spco = {}
|
||||
spcp = {}
|
||||
for row in rowsh:
|
||||
if row["species"] not in spco:
|
||||
spco[row["species"]]=0
|
||||
spcp[row["species"]]=0
|
||||
spco[row["species"]] = 0
|
||||
spcp[row["species"]] = 0
|
||||
|
||||
maxcount_o=int((len_p)*0.3/14)
|
||||
maxcount_p=int((len_p)*0.7/14)
|
||||
maxcount_o = int((len_p)*0.3/14)
|
||||
maxcount_p = int((len_p)*0.7/14)
|
||||
for row in rowsh:
|
||||
if row["label"]==2:
|
||||
if row["label"] == 2:
|
||||
continue
|
||||
if row["label"]==1 and spco[row["species"]]>maxcount_o:
|
||||
if row["label"] == 1 and spco[row["species"]] > maxcount_o:
|
||||
continue
|
||||
if row["label"]==0 and spcp[row["species"]]>maxcount_p:
|
||||
if row["label"] == 0 and spcp[row["species"]] > maxcount_p:
|
||||
continue
|
||||
bls.append(np.array(row["bls"]))
|
||||
blhs.append(np.array(row["blhs"]))
|
||||
|
|
@ -95,23 +127,23 @@ def read_positive(len_p,bls,blhs,dis,dps,dphs,sml,smg,pfam,label):
|
|||
smg.append(row["global_alignment_matrix"])
|
||||
pfam.append(row["pfam_matrix"])
|
||||
label.append(row["label"])
|
||||
if row["label"]==1:
|
||||
spco[row["species"]]+=1
|
||||
if row["label"]==0:
|
||||
spcp[row["species"]]+=1
|
||||
if row["label"] == 1:
|
||||
spco[row["species"]] += 1
|
||||
if row["label"] == 0:
|
||||
spcp[row["species"]] += 1
|
||||
|
||||
|
||||
def read_negative(len_n,bls,blhs,dis,dps,dphs,sml,smg,pfam,label):
|
||||
rows=[]
|
||||
with open("dataset","rb") as file:
|
||||
rows=pickle.load(file)
|
||||
rows=[row for row in rows if row["label"]==2]
|
||||
shi=np.random.permutation(len(rows))
|
||||
rows_shuffled=[]
|
||||
def read_negative(len_n, bls, blhs, dis, dps, dphs, sml, smg, pfam, label):
|
||||
rows = []
|
||||
with open("dataset", "rb") as file:
|
||||
rows = pickle.load(file)
|
||||
rows = [row for row in rows if row["label"] == 2]
|
||||
shi = np.random.permutation(len(rows))
|
||||
rows_shuffled = []
|
||||
for i in range(len(shi)):
|
||||
rows_shuffled.append(rows[shi[i]])
|
||||
rows=rows_shuffled
|
||||
rows=rows[:len_n]
|
||||
rows = rows_shuffled
|
||||
rows = rows[:len_n]
|
||||
for row in rows:
|
||||
bls.append(np.array(row["bls"]))
|
||||
blhs.append(np.array(row["blhs"]))
|
||||
|
|
@ -123,70 +155,73 @@ def read_negative(len_n,bls,blhs,dis,dps,dphs,sml,smg,pfam,label):
|
|||
label.append(row["label"])
|
||||
pfam.append(row["pfam_matrix"])
|
||||
|
||||
def train_models(model_name,start,end,num_epochs,learn_rate,decay,size_train,batch_size):
|
||||
k=1
|
||||
for i in range(start//10,end//10+1):
|
||||
bls=[]
|
||||
blhs=[]
|
||||
dis=[]
|
||||
dps=[]
|
||||
dphs=[]
|
||||
sml=[]
|
||||
smg=[]
|
||||
pfam=[]
|
||||
label=[]
|
||||
portion=float(i/10)
|
||||
len_n=int(size_train*portion)
|
||||
len_p=int(size_train*(1-portion))
|
||||
read_positive(len_p,bls,blhs,dis,dps,dphs,sml,smg,pfam,label)
|
||||
read_negative(len_n,bls,blhs,dis,dps,dphs,sml,smg,pfam,label)
|
||||
bls=create_branch_length_padding(bls)
|
||||
blhs=create_branch_length_padding(blhs)
|
||||
bls=np.array(bls)
|
||||
|
||||
def train_models(model_name, start, end, num_epochs, learn_rate,
|
||||
decay, size_train, batch_size):
|
||||
k = 1
|
||||
for i in range(start//10, end//10+1):
|
||||
bls = []
|
||||
blhs = []
|
||||
dis = []
|
||||
dps = []
|
||||
dphs = []
|
||||
sml = []
|
||||
smg = []
|
||||
pfam = []
|
||||
label = []
|
||||
portion = float(i/10)
|
||||
len_n = int(size_train*portion)
|
||||
len_p = int(size_train*(1-portion))
|
||||
read_positive(len_p, bls, blhs, dis, dps, dphs, sml, smg, pfam, label)
|
||||
read_negative(len_n, bls, blhs, dis, dps, dphs, sml, smg, pfam, label)
|
||||
bls = create_branch_length_padding(bls)
|
||||
blhs = create_branch_length_padding(blhs)
|
||||
bls = np.array(bls)
|
||||
print(bls.shape)
|
||||
blhs=np.array(blhs)
|
||||
blhs = np.array(blhs)
|
||||
print(blhs.shape)
|
||||
dis=np.array(dis)
|
||||
dis = np.array(dis)
|
||||
print(dis.shape)
|
||||
dps=np.array(dps)
|
||||
dps = np.array(dps)
|
||||
print(dps.shape)
|
||||
dphs=np.array(dphs)
|
||||
dphs = np.array(dphs)
|
||||
print(dphs.shape)
|
||||
sml=np.array(sml)
|
||||
sml = np.array(sml)
|
||||
print(sml.shape)
|
||||
smg=np.array(smg)
|
||||
smg = np.array(smg)
|
||||
print(smg.shape)
|
||||
pfam=np.array(pfam)
|
||||
pfam = np.array(pfam)
|
||||
print(pfam.shape)
|
||||
label=np.array(label)
|
||||
label = np.array(label)
|
||||
print(label.shape)
|
||||
shi=np.random.permutation(len(label))
|
||||
labels=label[shi]
|
||||
bls=bls[shi]
|
||||
blhs=blhs[shi]
|
||||
dis=dis[shi]
|
||||
dps=dps[shi]
|
||||
dphs=dphs[shi]
|
||||
sml=sml[shi]
|
||||
smg=smg[shi]
|
||||
pfam=pfam[shi]
|
||||
train(smg,sml,pfam,bls,blhs,dps,dphs,dis,labels,k,num_epochs,learn_rate,decay,int(0.9*size_train),batch_size,model_name)
|
||||
k+=1
|
||||
shi = np.random.permutation(len(label))
|
||||
labels = label[shi]
|
||||
bls = bls[shi]
|
||||
blhs = blhs[shi]
|
||||
dis = dis[shi]
|
||||
dps = dps[shi]
|
||||
dphs = dphs[shi]
|
||||
sml = sml[shi]
|
||||
smg = smg[shi]
|
||||
pfam = pfam[shi]
|
||||
train(smg, sml, pfam, bls, blhs, dps, dphs, dis, labels, k, num_epochs,
|
||||
learn_rate, decay, int(0.9*size_train), batch_size, model_name)
|
||||
k += 1
|
||||
|
||||
|
||||
def main():
|
||||
arg=sys.argv
|
||||
model_name=arg[-8]
|
||||
start_p=int(arg[-7])
|
||||
end_p=int(arg[-6])
|
||||
num_epochs=int(arg[-5])
|
||||
learn_rate=float(arg[-4])
|
||||
decay=float(arg[-3])
|
||||
size_train=float(arg[-2])
|
||||
batch_size=int(arg[-1])
|
||||
train_models(model_name,start_p,end_p,num_epochs,learn_rate,decay,size_train,batch_size)
|
||||
arg = sys.argv
|
||||
model_name = arg[-8]
|
||||
start_p = int(arg[-7])
|
||||
end_p = int(arg[-6])
|
||||
num_epochs = int(arg[-5])
|
||||
learn_rate = float(arg[-4])
|
||||
decay = float(arg[-3])
|
||||
size_train = float(arg[-2])
|
||||
batch_size = int(arg[-1])
|
||||
train_models(model_name, start_p, end_p, num_epochs,
|
||||
learn_rate, decay, size_train, batch_size)
|
||||
|
||||
|
||||
if __name__=="__main__":
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue