Skip to content

Commit

Permalink
Faster builds (#56)
Browse files Browse the repository at this point in the history
  • Loading branch information
@mariusandra
mariusandra committed Jan 24, 2024
1 parent b559114 commit 6c6d1e6
Show file tree
Hide file tree
Showing 2 changed files with 64 additions and 39 deletions.
64 changes: 25 additions & 39 deletions backend/app/tasks/deploy_frame.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ def install_if_necessary(package: str, raise_on_error = True) -> int:
uname_output = []
exec_command(frame, ssh, "uname -m", uname_output)
arch = "".join(uname_output).strip()
if arch == "aarch64":
if arch == "aarch64" or arch == "arm64":
cpu = "arm64"
elif arch == "armv6l" or arch == "armv7l":
cpu = "arm"
Expand Down Expand Up @@ -104,7 +104,7 @@ def install_if_necessary(package: str, raise_on_error = True) -> int:
log(id, "stdout", f"> add /srv/frameos/build/build_{build_id}.tar.gz")
scp.put(archive_path, f"/srv/frameos/build/build_{build_id}.tar.gz")
exec_command(frame, ssh, f"cd /srv/frameos/build && tar -xzf build_{build_id}.tar.gz && rm build_{build_id}.tar.gz")
exec_command(frame, ssh, f"cd /srv/frameos/build/build_{build_id} && sh ./compile_frameos.sh")
exec_command(frame, ssh, f"cd /srv/frameos/build/build_{build_id} && make -j$(nproc)")
exec_command(frame, ssh, f"mkdir -p /srv/frameos/releases/release_{build_id}")
exec_command(frame, ssh, f"cp /srv/frameos/build/build_{build_id}/frameos /srv/frameos/releases/release_{build_id}/frameos")
log(id, "stdout", f"> add /srv/frameos/releases/release_{build_id}/frame.json")
Expand Down Expand Up @@ -295,43 +295,29 @@ def create_local_build_archive(frame: Frame, build_dir: str, build_id: str, nim_
for file in files:
shutil.copy(os.path.join(source_dir, "src", "drivers", "waveshare", "ePaper", file), os.path.join(build_dir, file))

# Update the compilation script for verbose output
script_path = os.path.join(build_dir, "compile_frameos.sh")
log(frame.id, "stdout", f"Cleaning build script at {script_path}")
with open(script_path, "r") as file:
lines = file.readlines()
with open(script_path, "w") as file:
file.write("#!/bin/sh\n")
file.write("set -eu\n")
file.write("start_time=$(date +%s)\n")
file.write("mkdir -p ../cache\n") # make sure we have the cache folder
file.write("cached_files_count=0\n") # Initialize cached files counter
for i, line in enumerate(lines):
if line.startswith("gcc -c") and line.strip().endswith(".c"):
source_file = line.split(' ')[-1].strip()
o_file = line.split(' ')[-2].strip()
source_cleaned = '/'.join(source_file.split('@s')[-3:]).replace('@m', './')

# take the md5sum of the source file <source>.c
file.write(f"md5sum={compile_line_md5(line)}$(md5sum {source_file} | awk '{{print $1}}')\n")
# check if there's a file in the cache folder called <md5sum>.c.o
file.write(f"if [ -f ../cache/${{md5sum}}.{cpu}.c.o ]; then\n")
# if there is, make a symlink to the build folder with the name <source>.o
file.write(" cached_files_count=$((cached_files_count + 1))\n")
file.write(f" ln -s ../cache/${{md5sum}}.{cpu}.c.o {o_file}\n")
file.write("else\n")
# if not, run the command in "line" and then copy the <source>.c.o into the build folder as <md5sum>.c.o
file.write(f" echo [{i + 1}/{len(lines)}] Compiling on device: {source_cleaned}\n")
file.write(f" {line.strip()}\n")
file.write(f" cp {o_file} ../cache/${{md5sum}}.{cpu}.c.o\n")
file.write("fi\n")
else:
file.write(f"echo [{i + 1}/{len(lines)}] Compiling on device: frameos\n")
file.write(line)
file.write("echo \"Used $cached_files_count cached files\"\n")
file.write("end_time=$(date +%s)\n")
file.write("duration=$((end_time - start_time))\n")
file.write("echo \"Compiled in $duration seconds\"\n")
# Create Makefile
with open(os.path.join(build_dir, "Makefile"), "w") as file:
# Read the compilation flags from the generated script
script_path = os.path.join(build_dir, "compile_frameos.sh")
linker_flags = ["-pthread", "-lm", "-lrt", "-ldl"]
compiler_flags = []
with open(script_path, "r") as script:
lines = script.readlines()
for line in lines:
if " -o frameos " in line and " -l" in line:
linker_flags = [flag.strip() for flag in line.split(' ') if flag.startswith('-') and flag != '-o']
elif " -c " in line and len(compiler_flags) == 0:
compiler_flags = [flag for flag in line.split(' ') if flag.startswith('-') and not flag.startswith('-I') and not flag in ['-o', '-c']]

# Read the Makefile from ../frameos/tools/nimc.Makefile
with open(os.path.join(source_dir, "tools", "nimc.Makefile"), "r") as makefile:
lines = makefile.readlines()
for line in lines:
if line.startswith("LIBS = "):
line = "LIBS = -L. " + (" ".join(linker_flags)) + "\n"
if line.startswith("CFLAGS = "):
line = "CFLAGS = " + (" ".join([f for f in compiler_flags if f != '-c'])) + "\n"
file.write(line)

# 7. Zip it up "(cd tmp && tar -czf ./build_1.tar.gz build_1)"
archive_path = os.path.join(temp_dir, f"build_{build_id}.tar.gz")
Expand Down
39 changes: 39 additions & 0 deletions frameos/tools/nimc.Makefile
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
# This Makefile is used for compiling C sources generated by Nim
CC ?= gcc

SOURCES := $(shell ls -S *.c)
OBJECTS = $(SOURCES:.c=.o)
TOTAL = $(words $(SOURCES))
EXECUTABLE = frameos
LIBS = -pthread -lm -lm -lrt -ldl
CFLAGS = -w -fmax-errors=3 -pthread -O3 -fno-strict-aliasing -fno-ident -fno-math-errno

all: $(EXECUTABLE)

$(EXECUTABLE): $(OBJECTS)
@echo "Linking frameos"
@$(CC) -o $(EXECUTABLE) $(OBJECTS) $(LIBS)

clean:
rm -f *.o $(EXECUTABLE)

pre-build:
mkdir -p ../cache

$(OBJECTS): pre-build

%.o: %.c
@if [ ! -e $@ ]; then \
md5sum=$$(md5sum $< | awk '{print $1}'); \
file=$$(echo '$<' | sed 's/@s/\//g' | sed 's/@m//g' | sed 's/.*nimble\/pkgs2\/\(.*\)/\1/' | sed 's/.*\/\(nim\/lib\/.*\)/\1/'); \
cache_obj=../cache/$$md5sum.c.o; \
if [ -f "$$cache_obj" ]; then \
ln -s "$$cache_obj" $@; \
else \
$(CC) -c $(CFLAGS) $< -o $@; \
cp $@ "$$cache_obj"; \
echo "[$$(ls *.o | wc -l)/$(TOTAL)] $$file"; \
fi; \
fi

.PHONY: all clean pre-build

0 comments on commit 6c6d1e6

Please sign in to comment.