"make: execvp: /bin/sh: Argument list too long" error on standalone mode

Description of problem

When using standalone mode, “make: execvp: /bin/sh: Argument list too long” error occured.
I investigated my makefile generated by brian2, and found the “SRCS” argument was too long.
I want to split “SRCS” argument, but I am a newbe of c++, so I don’t know the proper way.
Someone knows how solve this error?

Minimal code to reproduce problem

#!/usr/bin/env python
# coding: utf-8

import time
from collections import defaultdict

import matplotlib.pyplot as plt
import neurokit2 as nk
import numpy as np
import pandas as pd
from brian2 import *
from smallworld import get_smallworld_graph
from smallworld.draw import draw_network
from tqdm import tqdm

import random

# スタンドアローンモードへ
set_device("cpp_standalone", build_on_run=False)
# prefs.devices.cpp_standalone.openmp_threads = 64

# イジケビッチニューロンの定義
eqs = Equations(
    """
dv/dt = (0.04/ms/mV)*v**2+(5/ms)*v+140*mV/ms-u+I : volt
du/dt = a*(b*v-u)                                : volt/second
I                                                : volt/second
a                                                : 1/second
b                                                : 1/second
c                                                : volt
d                                                : volt/second
"""
)

reset = """
v = c
u = u + d
"""


taupre = taupost = 20 * ms
# 論文の設定
wmax = 10
Apre = 0.1
Apost = -0.12

n = 1000  # number of neurons in one group
neuron_group_count = 100
R = 0.8  # ratio about excitory-inhibitory neurons
first_sim_ms = 1000 * 1000
second_sim_ms = 100 * 1000
third_sim_ms = 100 * 1000

# 各ニューロングループの生成
# 30mvでスパイクが発生する。数値積分法はeuler
P = NeuronGroup(n * neuron_group_count, model=eqs, threshold="v>30*mvolt", reset=reset, method="euler")
re = np.random.random(int(n * R))
ri = np.random.random(round(n * (1 - R)))

groups = []
# サブグループに分ける
for i in tqdm(range(neuron_group_count)):
    start = int(i * n)
    end = int((i + 1) * n)

    group = P[start:end]
    # 興奮性
    Pe = group[: int(n * R)]
    # 抑制性
    Pi = group[int(n * R) :]

    # 各種設定
    Pe.a = 0.02 / msecond  # 正
    Pe.b = 0.2 / msecond  # 正
    Pe.c = (15 * re ** 2 - 65) * mvolt
    Pe.d = (-6 * re ** 2 + 8) * mvolt / msecond
    Pe.I = 20 * mvolt / msecond  # 他が全部mVなのにこれだけvoltはおかしい /msecondじゃないと頻度が低すぎる
    # Pe.u = Pe.b * Pe.c
    # Pe.v = Pe.c

    Pi.a = (0.08 * ri ** 2 + 0.02) * 1 / msecond  # 正
    Pi.b = (-0.05 * ri ** 2 + 0.25) * 1 / msecond  # 正
    Pi.c = -65 * mvolt
    Pi.d = 2 * mvolt / msecond
    Pi.I = 20 * mvolt / msecond
    # Pi.u = Pi.b * Pi.c
    # Pi.v = Pi.c

    # グループ内の接続
    # 興奮性ニューロン to 同じニューロングループ内の100個のニューロンへのランダム接続
    Ce = Synapses(
        Pe,
        group,
        """
        w : 1
        dapre/dt = -apre/taupre : 1 (event-driven)
        dapost/dt = -apost/taupost : 1 (event-driven)
        """,
        on_pre="""
        v_post += w * mV
        apre += Apre
        w = clip(w+apost, 0, wmax)
        """,
        on_post="""
        apost += Apost
        w = clip(w+apre, 0, wmax)
        """,
    )
    Ce.connect(p=0.1)
    Ce.w = 6.0
    Ce.delay = round(random.uniform(0, 20), 2) * ms
    # 抑制性ニューロン to 同じニューロングループ内の100個の興奮性ニューロンへのランダム接続
    Ci = Synapses(
        Pi,
        Pe,
        """
        w : 1
        dapre/dt = -apre/taupre : 1 (event-driven)
        dapost/dt = -apost/taupost : 1 (event-driven)
        """,
        on_pre="""
        v_post += w * mV
        apre += Apre
        w = clip(w+apost, -wmax, 0)
        """,
        on_post="""
        apost += Apost
        w = clip(w+apre, -wmax, 0)
        """,
    )
    Ci.connect(p=0.125)
    Ci.w = -5.0
    Ci.delay = 1 * ms

    groups.append((group, Pe, Pi, Ce, Ci))

# WSモデルに従い、各グループの興奮性ニューロンから隣接する6つのノードへの接続と再配線を行う

# define network parameters
N = neuron_group_count
k_over_2 = 3
beta = 1.0
label = r"$\beta=0$"

focal_node = 0
# generate small-world graphs and draw
G = get_smallworld_graph(N, k_over_2, beta)
inter_synapses = []

for edge in tqdm(list(G.edges())):
    source_group_Pe = groups[edge[0]][1]
    target_group = groups[edge[1]][0]

    Ce = Synapses(
        source_group_Pe,
        target_group,
        """
        w : 1
        dapre/dt = -apre/taupre : 1 (event-driven)
        dapost/dt = -apost/taupost : 1 (event-driven)
        """,
        on_pre="""
        v_post += w *mV
        apre += Apre
        w = clip(w+apost, 0, wmax)
        """,
        on_post="""
        apost += Apost
        w = clip(w+apre, 0, wmax)
        """,
    )
    # 各興奮性ニューロンが、他のニューロングループと三本の接続を持つので、接続する確率は3/1000
    Ce.connect(p=0.003)
    Ce.w = 6.0
    Ce.delay = round(random.uniform(10, 30), 2) * ms
    inter_synapses.append(Ce)

net = Network([group[3] for group in groups], [group[4] for group in groups], inter_synapses, P)

# とりあええず1000秒動かす
value_interval_ms = 1
defaultclock.dt = value_interval_ms * ms
net.run(first_sim_ms * ms, report="stdout")

# STDPの設定を外す
for group in groups:
    group[3].pre.code = "v_post +=w* mV"
    group[3].post.code = ""
    group[4].pre.code = "v_post +=w *mV"
    group[4].post.code = ""

for inter in inter_synapses:
    inter.pre.code = "v_post +=w *mV"
    inter.post.code = ""

# 100秒動かす
net.run(second_sim_ms * ms, report="stdout")

# inputの設定を外す
new_I = array([0.0 for i in range(n * neuron_group_count)])
P.I = new_I * volt / second

V = StateMonitor(P, "v", record=True)
A = StateMonitor(P, "a", record=True)
B = StateMonitor(P, "b", record=True)
C = StateMonitor(P, "c", record=True)
D = StateMonitor(P, "d", record=True)
U = StateMonitor(P, "u", record=True)
S = SpikeMonitor(P)

net.add(V)
net.add(A)
net.add(B)
net.add(C)
net.add(D)
net.add(U)
net.add(S)

# 100秒動かす
net.run(third_sim_ms * ms, report="stdout")

# スタンドアローンモードへ
device.build(directory="output", compile=True, run=True, debug=False)

...(other codes)

What you have aready tried

I replaced the following code like “$(SRCS)” to “brianlib/randomkit/randomkit.c code_objects/after_run_neurongroup_thresholder_codeobject.cpp”

make.deps: $(SRCS) $(H_SRCS)
	$(CXX) $(CXXFLAGS) -MM $(SRCS) > make.deps

occured this error
“/bin/ld: cannot find code_objects/: file format not recognized”

Expected output (if relevant)

Actual output (if relevant)

Full traceback of error (if relevant)

make: execvp: /bin/sh: Argument list too long
make: *** [makefile:23: make.deps] Error 127
make: *** Waiting for unfinished jobs....
ERROR      Brian 2 encountered an unexpected error. If you think this is a bug in Brian 2, please report this issue either to the discourse forum at <http://brian.discourse.group/>, or to the issue tracker at <https://github.com/brian-team/brian2/issues>. Please include this file with debug information in your report: /JOBs/tmpdir/pbs.3410950.spcc-adm1/brian_debug_tobe2305.log  Additionally, you can also include a copy of the script that was run, available at: /JOBs/tmpdir/pbs.3410950.spcc-adm1/brian_script_c2u4hzqw.py You can also include a copy of the redirected std stream outputs, available at '/JOBs/tmpdir/pbs.3410950.spcc-adm1/brian_stdout_m4iq5bjv.log' and '/JOBs/tmpdir/pbs.3410950.spcc-adm1/brian_stderr_v50mvkzm.log'. Thanks! [brian2]
Traceback (most recent call last):
  File "reproduce_gachi.py", line 243, in <module>
    device.build(directory="output", compile=True, run=True, debug=False,clean=True)
  File "/home/s2030415/.local/lib/python3.8/site-packages/brian2/devices/cpp_standalone/device.py", line 1244, in build
    self.compile_source(directory, compiler, debug, clean)
  File "/home/s2030415/.local/lib/python3.8/site-packages/brian2/devices/cpp_standalone/device.py", line 989, in compile_source
    raise RuntimeError(error_message)
RuntimeError: Project compilation failed (error code: 512).

Hi @canonrock16 . In principle you could probably work around the limitation of the number of file names, but this would only be a workaround. The underlying problem is that your code generates many Synapses objects, and each of these objects is translated into several C++ files (one for setting up the connections, one for updating the variables, one of propagating spikes, …). This will take a very long time to compile and is not necessary in your case. The general rule should be that if several Synapses objects connect between the same NeuronGroups and use the same equations, then they can be represented by a single Synapses object. If I am not mistaken, in your code all Synapses use the same equations (STDP, etc.) and connect within the single NeuronGroup. This means that you’d only need a single Synapses object. The code might be clearer and easier to use to if you use instead a small number, but it should not be 100s of them as in your current code.

To give you an idea of how you can transform the code, let me illustrate with some simpler version of your code. Currently you do something like this which will create 2×100 Synapses objects:

n = 1000  # number of neurons in one group
neuron_group_count = 100
R = 0.8
P = NeuronGroup(n * neuron_group_count, ...)
for i in range(neuron_group_count):
    group = P[int(i * n):int((i + 1) * n)]
    Pe = group[:int(n * R)]
    Pi = group[int(n * R):]

    Ce = Synapses(Pe, group, ...)
    Ce.connect(p=0.1)
    Ce.w = 6.0  # single weight for all Pe→group connections
    Ce.delay = round(random.uniform(0, 20), 2) * ms  # single delay

    Ci = Synapses(Pi, Pe, ...)
    Ci.connect(p=0.125)
    Ci.w = -5.0
    Ci.delay = 1 * ms

One option is to calculate all the indices and delays by hand and then connect everything up in the end:

n = 1000  # number of neurons in one group
neuron_group_count = 100
R = 0.8
P = NeuronGroup(n * neuron_group_count, ...)
source_indices = []
target_indices = []
delays = []
weights = []
for i in range(neuron_group_count):
    group = np.arange(int(i * n), int((i + 1) * n))
    Pe = group[:int(n * R)]
    Pi = group[int(n * R):]

    ci, cj = np.meshgrid(Pe, group, indexing='ij')  # all possible combinations of neurons in Pe and group
    select = np.random.rand(len(Pe), len(group)) < 0.1  # which pairs should be selected?
    n_select = np.sum(select)  # number of selected pairs
    
    # Store selected indices
    source_indices.extend(ci[select])
    target_indices.extend(cj[select])

    # Store corresponding weights/delays
    delays.extend(np.repeat(round(random.uniform(0, 20), 2), n_select)*ms)
    weights.extend(np.repeat(6.0, n_select))
 
    # Same thing for inhibitory connections
    ci, cj = np.meshgrid(Pi, Pe, indexing='ij')
    select = np.random.rand(len(Pi), len(Pe)) < 0.125
    n_select = np.sum(select)  # number of selected pairs
    source_indices.extend(ci[select])
    target_indices.extend(cj[select])
    delays.extend(np.repeat(1, n_select) * ms)
    weights.extend(np.repeat(-5.0, n_select))

# Now, create a *single* Synapses object storing everything
C = Synapses(P, P, '''...''', ...)
C.connect(i=source_indices, j=target_indices)
C.w = weights
C.delay = delays

You could of course clear this up a bit by putting some of the numpy calculation into a separate function, but I hope that the general idea is clear (if not, try it out with a small network and see what the variables contain at each step). Another option is slightly less efficient for the synapse creation, but more compact to write. It also makes the idea behind the connections much clearer, I think.

n = 1000  # number of neurons in one group
neuron_group_count = 100
R = 0.8
# Each neuron has a group label (integer number), and a flag whether it is excitatory or not
P = NeuronGroup(n * neuron_group_count, '''...
                                           group : integer (constant)
                                           is_excitatory : boolean (constant)
                    ''')
P.group = 'i // n'  # 0 for the first 1000 neurons, then 1 for the next 1000 neurons, etc.
P.is_excitatory = '(i % n) < int(R*n)'

C = Synapses(P, P, ...)
# excitatory connections
C.connect('is_excitatory_pre', p=0.1) 
C.delay = 'int(rand*2000)/100*ms'
C.weights = 6
# inhibitory connections
C.connect('not is_excitatory_pre and is_excitatory_post and group_pre == group_post', p=0.125)
C.w['not is_excitatory_pre'] = -5.0
C.delay['not is_excitatory_pre'] = 1*ms

There is a small difference between this and the previous code: here each excitatory connection has a different delay, whereas previously there was one delay value per group. This would be a bit more complicated to express in this framework. As I said before, you might also decide that it is more straightforward to have separate Ce and Ci synapses here instead of merging everything.

Hope that makes things a bit clearer! Best
Marcel

@mstimberg
Thank you for your help!
Sorry for late reply.I had been tried your solution, but that experiments took a lot of time.
Finally I could run my script on standalone mode.Thank you!

1 Like