From e76c3c794f7bd79c2812ff6d3aefc5a53cbc4b8d Mon Sep 17 00:00:00 2001 From: Saksham Gera Date: Wed, 27 Aug 2025 02:52:02 +0530 Subject: [PATCH] Added The Comments in Both concore.py and mkconcore.py --- concore.py | 71 ++++++++++++++--- .../Throughput/maximumThroughputComaprison.py | 1 - .../Throughput/throughput_comparison.pdf | Bin 14603 -> 13263 bytes mkconcore.py | 74 ++++++++++++++++-- 4 files changed, 129 insertions(+), 17 deletions(-) diff --git a/concore.py b/concore.py index ce3ab0a..6d71f0f 100644 --- a/concore.py +++ b/concore.py @@ -5,26 +5,35 @@ import re import zmq # Added for ZeroMQ -#if windows, create script to kill this process +# if windows, create script to kill this process # because batch files don't provide easy way to know pid of last command -# ignored for posix!=windows, because "concorepid" is handled by script -# ignored for docker (linux!=windows), because handled by docker stop +# ignored for posix != windows, because "concorepid" is handled by script +# ignored for docker (linux != windows), because handled by docker stop if hasattr(sys, 'getwindowsversion'): with open("concorekill.bat","w") as fpid: fpid.write("taskkill /F /PID "+str(os.getpid())+"\n") -# --- ZeroMQ Integration Start --- +# =================================================================== +# ZeroMQ Communication Wrapper +# =================================================================== class ZeroMQPort: def __init__(self, port_type, address, zmq_socket_type): + """ + port_type: "bind" or "connect" + address: ZeroMQ address (e.g., "tcp://*:5555") + zmq_socket_type: zmq.REQ, zmq.REP, zmq.PUB, zmq.SUB etc. + """ self.context = zmq.Context() self.socket = self.context.socket(zmq_socket_type) self.port_type = port_type # "bind" or "connect" self.address = address - self.socket.setsockopt(zmq.RCVTIMEO, 2000) - self.socket.setsockopt(zmq.SNDTIMEO, 2000) - self.socket.setsockopt(zmq.LINGER, 0) + # Configure timeouts & immediate close on failure + self.socket.setsockopt(zmq.RCVTIMEO, 2000) # 2 sec receive timeout + self.socket.setsockopt(zmq.SNDTIMEO, 2000) # 2 sec send timeout + self.socket.setsockopt(zmq.LINGER, 0) # Drop pending messages on close + # Bind or connect if self.port_type == "bind": self.socket.bind(address) print(f"ZMQ Port bound to {address}") @@ -33,6 +42,7 @@ def __init__(self, port_type, address, zmq_socket_type): print(f"ZMQ Port connected to {address}") def send_json_with_retry(self, message): + """Send JSON message with retries if timeout occurs.""" for attempt in range(5): try: self.socket.send_json(message) @@ -44,6 +54,7 @@ def send_json_with_retry(self, message): return def recv_json_with_retry(self): + """Receive JSON message with retries if timeout occurs.""" for attempt in range(5): try: return self.socket.recv_json() @@ -89,6 +100,9 @@ def terminate_zmq(): print(f"Error while terminating ZMQ port {port.address}: {e}") # --- ZeroMQ Integration End --- +# =================================================================== +# File & Parameter Handling +# =================================================================== def safe_literal_eval(filename, defaultValue): try: with open(filename, "r") as file: @@ -97,10 +111,13 @@ def safe_literal_eval(filename, defaultValue): # Keep print for debugging, but can be made quieter # print(f"Info: Error reading {filename} or file not found, using default: {e}") return defaultValue - + + +# Load input/output ports if present iport = safe_literal_eval("concore.iport", {}) oport = safe_literal_eval("concore.oport", {}) +# Global variables s = '' olds = '' delay = 1 @@ -110,14 +127,20 @@ def safe_literal_eval(filename, defaultValue): simtime = 0 #9/21/22 +# =================================================================== +# Parameter Parsing +# =================================================================== try: sparams_path = os.path.join(inpath + "1", "concore.params") if os.path.exists(sparams_path): with open(sparams_path, "r") as f: sparams = f.read() if sparams: # Ensure sparams is not empty + # Windows sometimes keeps quotes if sparams[0] == '"' and sparams[-1] == '"': #windows keeps "" need to remove sparams = sparams[1:-1] + + # Convert key=value;key2=value2 to Python dict format if sparams != '{' and not (sparams.startswith('{') and sparams.endswith('}')): # Check if it needs conversion print("converting sparams: "+sparams) sparams = "{'"+re.sub(';',",'",re.sub('=',"':",re.sub(' ','',sparams)))+"}" @@ -137,11 +160,16 @@ def safe_literal_eval(filename, defaultValue): #9/30/22 def tryparam(n, i): + """Return parameter `n` from params dict, else default `i`.""" return params.get(n, i) #9/12/21 +# =================================================================== +# Simulation Time Handling +# =================================================================== def default_maxtime(default): + """Read maximum simulation time from file or use default.""" global maxtime maxtime_path = os.path.join(inpath + "1", "concore.maxtime") maxtime = safe_literal_eval(maxtime_path, default) @@ -149,6 +177,7 @@ def default_maxtime(default): default_maxtime(100) def unchanged(): + """Check if global string `s` is unchanged since last call.""" global olds, s if olds == s: s = '' @@ -156,16 +185,21 @@ def unchanged(): olds = s return False +# =================================================================== +# I/O Handling (File + ZMQ) +# =================================================================== def read(port_identifier, name, initstr_val): global s, simtime, retrycount + # Default return default_return_val = initstr_val if isinstance(initstr_val, str): try: default_return_val = literal_eval(initstr_val) except (SyntaxError, ValueError): pass - + + # Case 1: ZMQ port if isinstance(port_identifier, str) and port_identifier in zmq_ports: zmq_p = zmq_ports[port_identifier] try: @@ -178,6 +212,7 @@ def read(port_identifier, name, initstr_val): print(f"Unexpected error during ZMQ read on port {port_identifier} (name: {name}): {e}. Returning default.") return default_return_val + # Case 2: File-based port try: file_port_num = int(port_identifier) except ValueError: @@ -197,6 +232,7 @@ def read(port_identifier, name, initstr_val): print(f"Error reading {file_path}: {e}. Using default value.") return default_return_val + # Retry logic if file is empty attempts = 0 max_retries = 5 while len(ins) == 0 and attempts < max_retries: @@ -214,6 +250,8 @@ def read(port_identifier, name, initstr_val): return default_return_val s += ins + + # Try parsing try: inval = literal_eval(ins) if isinstance(inval, list) and len(inval) > 0: @@ -230,8 +268,13 @@ def read(port_identifier, name, initstr_val): def write(port_identifier, name, val, delta=0): + """ + Write data either to ZMQ port or file. + `val` must be list (with simtime prefix) or string. + """ global simtime + # Case 1: ZMQ port if isinstance(port_identifier, str) and port_identifier in zmq_ports: zmq_p = zmq_ports[port_identifier] try: @@ -240,7 +283,8 @@ def write(port_identifier, name, val, delta=0): print(f"ZMQ write error on port {port_identifier} (name: {name}): {e}") except Exception as e: print(f"Unexpected error during ZMQ write on port {port_identifier} (name: {name}): {e}") - + + # Case 2: File-based port try: if isinstance(port_identifier, str) and port_identifier in zmq_ports: file_path = os.path.join("../"+port_identifier, name) @@ -251,8 +295,9 @@ def write(port_identifier, name, val, delta=0): print(f"Error: Invalid port identifier '{port_identifier}' for file operation. Must be integer or ZMQ name.") return + # File writing rules if isinstance(val, str): - time.sleep(2 * delay) + time.sleep(2 * delay) # string writes wait longer elif not isinstance(val, list): print(f"File write to {file_path} must have list or str value, got {type(val)}") return @@ -269,6 +314,10 @@ def write(port_identifier, name, val, delta=0): print(f"Error writing to {file_path}: {e}") def initval(simtime_val_str): + """ + Initialize simtime from string containing a list. + Example: "[10, 'foo', 'bar']" → simtime=10, returns ['foo','bar'] + """ global simtime try: val = literal_eval(simtime_val_str) diff --git a/measurements/Throughput/maximumThroughputComaprison.py b/measurements/Throughput/maximumThroughputComaprison.py index 1dd0342..607d259 100644 --- a/measurements/Throughput/maximumThroughputComaprison.py +++ b/measurements/Throughput/maximumThroughputComaprison.py @@ -15,7 +15,6 @@ # Add plot details plt.ylabel('Throughput (Messages/Second)', fontsize=14) -plt.title('Figure 6: Maximum Throughput Comparison', fontsize=18, pad=25) plt.xticks(fontsize=12) plt.yscale('log') # log scale for large differences plt.grid(axis='y', linestyle='--', alpha=0.7) diff --git a/measurements/Throughput/throughput_comparison.pdf b/measurements/Throughput/throughput_comparison.pdf index 7793b139f42b8e6f99a2c3de5ee55e21457903ff..5854b05a0b0dfb015fa5f41eb267ef11eeff92fd 100644 GIT binary patch delta 2741 zcmZWqX;f3!7JdO_hzx-Q5eRrNfKt)i`Q{d|iUNXF6mhDEh7bl30!c&)`l6KjVDVa% zv#M6bMWIelt4OLi02NqbL@#zt)`8NXpkNqdi#Mafz3K#D_*!k%6@(AB` zd5YYvYf^L+l{FY1H{IJ^kr@7p3|DN)pS5(c&uo`GrnpR#iiL=4p1$%S0vRhnN=Ja>{{6MfdRj;JU25A9|KwEgANbP5Rl1IGR?n z@WuL$WL?3HGw{^0k&$f<51)2Tub+A_w^u~p{HXNd{QR%GCs?*!^lw{#eCi^%c&FFA zT*oM_<1}QLnQS-sU0q*$OI;s1U=zCFf%TDJ@A|gB{K%r=-SoNz-sd{D+(xX*>uSl# zB^Q{@2p9w$y zhU0=$tCn4tonUjleAG5!8L~3FK!FZZ_;yzhh0^JonH}!KM{l;&FKCI`-&Lsq^daYe!zSg}rk7uD_vS|BHTkM(u@jIlAGIid4tT zwg0QWd${ZA#u-B|eRD?of{*pbZm5ck-LO3F*8P*P6Z_$4!QnS){a%qb(tkhmBs|(< z@!Epf?Nsd2_Wu0^M>e-Tt07YE!?S)vQJcSNI_wt~S9<5!in}XC5wC69PrA_W%=|C7 zMevnr@fxjKDNp5)DsBTG#Q*{f0HJX>$r6F~)?dN|E0Mm!n&80^D-rAx`RG0P2|QS0 zErh!#Ziau@2qCcj41P9Is6T6)%!78eLOo{h$%Ey#LipU_Q#i%mS)VI#=fNcoLVbnf zEFSE35WxyZAH9pvh6gtaM34{_!vl^YeV3?^2X8qEVb-KB=p_18{yBRE<;Q@eD4j2YV6+jvfq)LDyIFKgtrAa`V3{Vt--80FW51B_GCvCAX^%pk|v!;Nyu05{5|0o*930=PjANL6V{fE&fq0d5fE zOVj0QW!gq%f)?OLQ3In+PE2AcqqG{}#ws-cXF!~x5KVFxAPmZUsb-^GlLQE(x+Yz& zPy)i}Knn53xXynm_~@r z2nPO}km^uI#;(CVAPw*h07Yqn?ZOWmf}?CLV=_?$MaH{B7>XO1v2iF)jpq=QL6{i9?bg$8Rt~8r${08Iqw*%upzISjL*62yRHnag?zT-FwwhGUol( z7*c4|bnj@AJDXz{q#4sV8Dd(94CT()-?=L}D8mVpF34!gq+JYRGOP%Lb63XS3$h5s z5CUG1iAnZANuyLMEnp+Kn=3pmfjyd|7s2e=e2rGF)@G@d3G9~zLvX&2Pe^1a|Nldq BlivUU delta 3824 zcmZuzdpwlc8*d1i+{TQ^^|(iB-kEoMO^DubAHMqDVQ{OTwJ-^*Vc#xl$RNCeCp}(%8Iq*LBstvE3S$}*;^up>dEtATQ z&Pu{i1KzrwD68Y2c;%t6Psh#7#VPriW<~u6fg{^L`E1T@$Nb8{{Ut|s+O0XGr97^- z;_(h{N=`#ieSX>orSJy%qPi0m!4~naZT6Liq`124_YX!xzEM$!bLydRN8$Q6*8N7= zEq50#tvJ4Om50UKd-l$hroh#v`-TXLLQ+~q>>_8Hhi|wbqWWx&C~@<;M)ktzR;s{ ziQGZwBXI*&rGDi}Kb|)>@F-Z<6BkdgEFp*UxmWiCFOFVfF?ANprEg;@izP z?id>yHMhSj8BWFLrtUf4otb>f+$$a++Y2@wPkwmI__mQ&R$^Zw?`2*R*T>Ot_@j?N zEiTn(;J^*$>10P42ev(V<1?bAjk-)@i>gZG?X@g_Unj52E$-Lv$l^?WWDHYoDhqAY z&*$PN9hRp#%8aVxwbBw)wc}0VBP&;48zyXhnG6_3ifg!$9E^QFpsh2u{I0tL$K!uV5Zt<> z@}#7pst)xg#J|&OYpBdz-CNf1^of4xDZOnSZtp2a2*+HO_?ZdgxIqm!5yrEJzm%2S z)YJ3{*xdXFLBo!05illK5K`bR*n3M+ufzXSt71v9&!#hv))mw~f6~6X-h8<6W2?1h zQ^vYj<2TQ|cGkaNp5Zt0s6nXNrrMSOCoPX$?iKQPO!RiW@0=W!moI;}ec6QS7Y4He z&*25}1<@S;2rPnoP32*;oT4lRrNIt4M3#!u;UkifkiK>mZx&@v&0CGhWhc&?4;GjA zl}&uTl)jcME8E^~GI-m;#P;sK_X{@s(AY<7`e5zdb+5(wRab2${|8tMp8*LvC zTQ)jAeQve&ihIXn=jTU523GmTZeAS`=BE`yV#?*I7+7Cd7&CffwRlM$Z}gBwM&WTo z)}kbXtTL|_m2(8MoxvvYjiv5`$7t6_?OI1_ci1`d%z`YhtZ}d`t)%_3r|ff1sM(pp zugRq&wKpbbt#I?Y29r&R@PIr~hRMJ>H(o_&u0z(Gzc@?%WcnNY8;AXPi*(;*wwZ-Z za>OQ_=)UXyB&!uJLHo0+4fk;Z{L}AJ8P!RRMjPdqX>;9F?hor$DOJ&C(KYAGNeaJI zAVNj?y)aRcB&v|7;h+MZ1nt#LMOt%$aj+Cmgn5c_P+N%zb?_w>K=BYceOGJINc9aFnG` zWtOo}_P%^r+G&Mq(3i6)r_7RImC2RgpgrqUz2-T{P&0#@mAAue0#VebtcQaXbt3dt zGZigWRm8yuM3Tr?Z7~kkkVvqUkPlrnNFsanY#e0GBf?JNZ_r1R2wO;ra8#2dLuX0M zvNF#ldN!D2|EYABanhGA8|ciTT>HvUeh|cnEjm4s>TG;c-RdoAJCfM&u{r2+ZybBQ zwT>KV<$?nZ>w2nXDy%*47KdkT?GNO1U2V0Nzo+wH6Z=e1)UiD=UNM&2Tw4l9);=GL z^)5(sRN>&;EBU(ImyHoWYneM7I{RMbQzf$rq&7Q+u>I1J|$G%P861kn=)S*_NXQ-vOE4IYJk3n_REH>1)H1ma4ilqk#XldIt3sgGGHs-#s-jGIlK@-C}6R_O|#oILK33X|6Q8ntz4(; zsqCwvA{Y5Uy|6B-r5yKuW+<&X{kI!d0OdTFkz<+zie-Cj9F!-V+rIjaxlKWaqq zO^F@euE#`nO~Ys3MW*QgwDkF2UHp`%BuGq6r_R(srHHajgfbd*79a;kawB_(=<^T$U0&)l-hXQgK_5)jq0AwB@M*{LTK%vk8IU11p zfGhyy7(k(7b>c;v&9&r&D*%Ow3Gl!Q0}AU~Bou=rym;))Xi{E2phG2#w}o zxKx86E)vkBDq_HVx|BU0NX&?xY62%Z640rlr;8YvSy8A$PBlVN8har^H>a==#M}bd z+KPQ7c29gXCm4?aIt^YVlS5Fd_!`6?-~qfOpitQ?jBP50BI3)OhEW**fw8dur{z%* z=!{s)Fc>fcF%{Cq-A~P+(2*ItFc~xQ5GE5l)2Vx@bo%r>ghFRwXF4Shp|IGJPKEz! z6`>;H>zbNK#lRUDU2Lamd2I2;O~p`zHDfPI72EFbJd7XvPzE+(lIZVqM^LtSh{WGN z7!6^}$fNy-KpGvv+%j!1OFSA=F*-#kem1RPfFwaiQ!(QHRLudT2%@l+-Lpuefvfxfs nSCfvt#vYN80w8&10C^QJ7#nyLR-?qZD2;~VO-)y>cEJA+pm#}D diff --git a/mkconcore.py b/mkconcore.py index c05e8aa..b243c23 100644 --- a/mkconcore.py +++ b/mkconcore.py @@ -1,3 +1,67 @@ +# The script handles different environments: Docker, POSIX (macOS/Ubuntu), and Windows. +# It reads the graph nodes (representing computational tasks) and edges (representing data flow). +# Based on this information, it generates a directory structure and a set of helper scripts +# (build, run, stop, clear, maxtime, params, unlock) to manage the workflow. +# It also includes logic to handle "script specialization" for ZMQ-based communication, +# where it modifies source files to include specific port and port-name information. + +# The script does the following: +# 1. Initial Setup and Argument Parsing: +# - Defines global constants for tool names (g++, iverilog, python3, matlab, etc.) and paths. +# - Parses command-line arguments for the GraphML file, source directory, output directory, and execution type (posix, windows, docker). +# - Checks for the existence of input/output directories and creates the output structure. +# - Logs the configuration details. + +# 2. Graph Parsing and Adjacency Matrix Creation: +# - Uses BeautifulSoup to parse the input GraphML file. +# - Identifies nodes and edges, storing them in dictionaries. +# - Creates a simple adjacency matrix (m) and a reachability matrix (ms) from the graph, +# detecting any unreachable nodes and logging a warning. + +# 3. Script Specialization (Aggregation and Execution): +# - This is a key part of the logic that handles ZMQ connections. +# - It iterates through the edges, specifically looking for ones with labels in the format "0x_". +# - It aggregates these port parameters for each node. +# - It then uses an external script `copy_with_port_portname.py` to "specialize" the original source files. This means it creates +# new versions of the scripts, injecting the ZMQ port information directly into the code. +# - The `nodes_dict` is then updated to point to these newly created, specialized scripts. + +# 4. Port Mapping and File Generation: +# - Generates `.iport` (input port) and `.oport` (output port) mapping files for each node. +# - These files are simple dictionaries that map volume names (for file-based communication) or port names (for ZMQ) +# to their corresponding port numbers or indices. This allows the individual scripts to know how to connect to their +# peers in the graph. + +# 5. File Copying and Script Generation: +# - Copies all necessary source files (`.py`, `.cpp`, `.m`, etc.) from the source directory to the `outdir/src` directory. +# - Handles cases where specialized scripts were created, ensuring the new files are copied instead of the originals. +# - Copies a set of standard `concore` files (`.py`, `.hpp`, `.v`, `.m`, `mkcompile`) into the `src` directory. + +# 6. Environment-Specific Scripting (Main Logic Branches): +# - This is the largest and most complex part, where the script's behavior diverges based on the `concoretype`. + +# a. Docker: +# - Generates `Dockerfile`s for each node's container. If a custom Dockerfile exists in the source directory, it's used. +# Otherwise, it generates a default one based on the file extension (`.py`, `.cpp`, etc.). +# - Creates `build.bat` (Windows) or `build` (POSIX) scripts to build the Docker images for each node. +# - Creates `run.bat`/`run` scripts to launch the containers, setting up the necessary shared volumes (`-v`) for data transfer. +# - Creates `stop.bat`/`stop` and `clear.bat`/`clear` scripts to manage the containers and clean up the volumes. +# - Creates helper scripts like `maxtime.bat`/`maxtime`, `params.bat`/`params`, and `unlock.bat`/`unlock` to +# pass runtime parameters or API keys to the containers. + +# b. POSIX (Linux/macOS) and Windows: +# - These branches handle direct execution on the host machine without containers. +# - Creates a separate directory for each node inside the output directory. +# - Uses the `build` script to copy source files and create symbolic links (`ln -s` on POSIX, `mklink` on Windows) +# between the node directories and the shared data directories (representing graph edges). +# - Generates `run` and `debug` scripts to execute the programs. It uses platform-specific commands +# like `start /B` for Windows and `xterm -e` or `osascript` for macOS to run the processes. +# - The `stop` and `clear` scripts use `kill` or `del` commands to manage the running processes and files. +# - Generates `maxtime`, `params`, and `unlock` scripts that directly write files to the shared directories. + +# 7. Permissions: +# - Sets the executable permission (`stat.S_IRWXU`) for the generated scripts on POSIX systems. + from bs4 import BeautifulSoup import logging import re @@ -17,13 +81,13 @@ CPPEXE = "g++" #Ubuntu/macOS C++ 6/22/21 VWIN = "iverilog" #Windows verilog 6/25/21 VEXE = "iverilog" #Ubuntu/macOS verilog 6/25/21 -PYTHONEXE = "python3" #Ubuntu/macOS python 3 -PYTHONWIN = "python" #Windows python 3 +PYTHONEXE = "python3" #Ubuntu/macOS python3 +PYTHONWIN = "python" #Windows python3 MATLABEXE = "matlab" #Ubuntu/macOS matlab MATLABWIN = "matlab" #Windows matlab -OCTAVEEXE = "octave" #Ubuntu/macOS octave -OCTAVEWIN = "octave" #Windows octave -M_IS_OCTAVE = False #treat .m as octave +OCTAVEEXE = "octave" #Ubuntu/macOS octave +OCTAVEWIN = "octave" #Windows octave +M_IS_OCTAVE = False #treat .m as octave MCRPATH = "~/MATLAB/R2021a" #path to local Ubunta Matlab Compiler Runtime DOCKEREXE = "sudo docker"#assume simple docker install DOCKEREPO = "markgarnold"#where pulls come from 3/28/21