I am trying to parallelise a network model I had used previously, as we are now in the lucky situation to have a beowulf cluster available for simulations.
I have adapted my network model based on a few examples posted in ModelDB (especially the parallel olfactory bulb network model.)
I am currently in the process of trying to reproduce the exact same results on the cluster that I get on my serial machine. Many parameters of my network are randomised, so I thought the differences I get so far are due to the multiple initialisations of random processes on the cluster nodes (in opposite to a single initialization on the serial machine).
To avoid this I simply set different highindex values (using MCellRan4) each time a cell is created on a node (A vector holds all highindex values, where the actual value is selected based on id of the the cell). And I set a different seed for the generation of the connections in the network (where exactly the same code is executed on every node).
Unfortunately this does not work. I do not get the same results and I am running out of ideas why that could be.
If I look at the raster plot (view) I can see that "neighbouring cells" (consecutive id's) have exactly similar spike times at the beginning of the simulation, which indicates that the parameter of the cells seem to be the same, hence my randomisation does not work properly!
My questions are:
Why does the initialisation (code below) not work, especially why do the neighbouring cells have similar parameter settings?
I am testing the parallel code with 10 active nodes, and 10 consecutive cells seem to have similar parameters. How does roundrobin distribute the cells over the nodes exactly?
Many thanks for any ideas or answers!
Ullrich
the respective code is here:
top level script:
Code: Select all
load_file("nrngui.hoc")
create acell_home_
access acell_home_
xopen("p_DA2Par.hoc")
tend=100
load_file("p_IB.tem")
load_file("p_IN.tem")
//1) load the netparmpi.hoc file and create a ParallelNetManager
load_file("netparmpi.hoc")
objref pnm, file, fo, ranind
ncell = Npc+Nin
pnm = new ParallelNetManager(ncell)
if (pnm.myid==0) print "nhosts\n" , pnm.pc.nhost()
nhosts=pnm.pc.nhost()
ranind=new Vector()
for i=0,ncell-1 { ranind.append((i+100)*1000) }
//2) distribute cells to nodes
pnm.round_robin()
//3 creation & connection
load_file("p_DA2Net.hoc")
load_file("perfrun.hoc")
// chk max delay
pnm.set_maxstep(100)
pnm.want_all_spikes()
// init & run
tstop=tend
v_init=vstart
objref cv
cv=new CVode(1)
//cv.active(1)
objref mxhist
if (pnm.myid == 0) {
mxhist = new Vector(11)
pnm.pc.max_histogram(mxhist)
}
//------------------------
prun()
//------------------------
//get results & stats
pnm.prstat(1)
pnm.gatherspikes()
getstat()
for i=0, mxhist.size-1 {
printf("%d\t %d\n", i, mxhist.x[i])
}
//printf("setuptime=%g runtime=%g\n", setuptime, runtime)
// shut down nodes
pnm.pc.runworker
pnm.pc.done
// print results
fo = new File()
fo.wopen("data/out2.dat")
for i=0, pnm.spikevec.size-1 {
//print pnm.spikevec.x[i], pnm.idvec.x[i]
fo.printf("%-10.6lf %d \n", pnm.spikevec.x[i], pnm.idvec.x[i])
}
fo.close()
//quit()
Code: Select all
objref randn
randn=new Random()
//randn.MCellRan4(1)
objref IB, IN
//for i=0,Npc*Ncol-1 {
obfunc newIB(){
// many random calls for randomisation of conductance
//eg
y=IB.soma.gHVAbar_HVA
IB.soma.gHVAbar_HVA=randn.normal(y,(y*gHVAstd)^2)
if (IB.soma.gHVAbar_HVA<0>PC connections
objref PCPCconA//[Npc*Ncol][Npc*Ncol]
objref PCPCconN//[Npc*Ncol][Npc*Ncol]
objref ConPCPC
ConPCPC = new Matrix(Npc*Ncol,Npc*Ncol)
PCPCconA= new Matrix(Npc*Ncol,Npc*Ncol)
PCPCconN= new Matrix(Npc*Ncol,Npc*Ncol)
randn.MCellRan4(3e+09)
for i=0,Npc*Ncol-1 {
for j=0,Npc*Ncol-1 {
q=randn.uniform(0,1)
if (int(i/Npc)==int(j/Npc)) { p=pconPPwc } else { p=pconPPbc }
ConPCPC.x[i][j]=-100
if (p>q) {
wgt=randn.normal(wPPavg,wPPstd^2)
if (wgt<0> in .tem connect2target
//IBnet[i].soma PCPCconA[i][j]=new NetCon(&v(.5),IBasyn[j],thPC,delPC,wgt)
//IBnet[i].soma PCPCconN[i][j]=new NetCon(&v(.5),IBnsyn[j],thPC,delPC,wgt)
ConPCPC.x[i][j]=wgt
}
}
}
// IN->IN connections
objref ININconG//[Nin*Ncol][Nin*Ncol]
objref ConININ
ConININ = new Matrix(Nin*Ncol,Nin*Ncol)
ININconG = new Matrix(Nin*Ncol,Nin*Ncol)
for i=0,Nin*Ncol-1 {
for j=0,Nin*Ncol-1 {
q=randn.uniform(0,1)
if (int(i/Nin)==int(j/Nin)) { p=pconIIwc } else { p=pconIIbc }
ConININ.x[i][j]=-100
if (p>q) {
wgt=randn.normal(wIIavg,wIIstd^2)
if (wgt<0>IN connections
objref PCINconA //[Npc*Ncol][Nin*Ncol]
objref PCINconN //[Npc*Ncol][Nin*Ncol]
objref ConPCIN
ConPCIN = new Matrix(Npc*Ncol,Nin*Ncol)
PCINconA = new Matrix(Npc*Ncol,Nin*Ncol)
PCINconN = new Matrix(Npc*Ncol,Nin*Ncol)
for i=0,Npc*Ncol-1 {
for j=0,Nin*Ncol-1 {
q=randn.uniform(0,1)
if (int(i/Npc)==int(j/Nin)) { p=pconPIwc } else { p=pconPIbc }
ConPCIN.x[i][j]=-100
if (p>q) {
wgt=randn.normal(wPIavg,wPIstd^2)
if (wgt<0>PC connections
objref INPCconG0//[Nin*Ncol][Npc*Ncol]
objref INPCconG1//[Nin*Ncol][Npc*Ncol]
objref ConINPC
ConINPC = new Matrix(Nin*Ncol,Npc*Ncol)
INPCconG0= new Matrix(Nin*Ncol,Npc*Ncol)
INPCconG1= new Matrix(Nin*Ncol,Npc*Ncol)
for i=0,Nin*Ncol-1 {
for j=0,Npc*Ncol-1 {
q=randn.uniform(0,1)
if (int(i/Nin)==int(j/Npc)) { p=pconIPwc } else { p=pconIPbc }
ConINPC.x[i][j]=-100
if (p>q) {
wgt=randn.normal(wIPavg,wIPstd^2)
if (wgt<0) wgt=0
INPCconG0.x[i][j]=pnm.nc_append(INid.x[i], PCid.x[j],0, wgt,delIN)
INPCconG1.x[i][j]=pnm.nc_append(INid.x[i], PCid.x[j],1, wgt,delIN)
//IN.soma INPCconG0[i][j]=new NetCon(&v(.5),IBg0syn[j],thIN,delIN,wgt)
//IN.soma INPCconG1[i][j]=new NetCon(&v(.5),IBg1syn[j],thIN,delIN,wgt)
ConINPC.x[i][j]=wgt
}
}
}