microbit v1 support (#4373)

* Fix handling of ImageLiteral_ typedef

* Put source outside flashing area in hex files

* Stop building dal.d.ts randomly

* Fix statement count calculation

* Squash consecutive pushes and consecitve pops

cli/buildengine.ts

@@ -330,15 +330,16 @@ function updateCodalBuildAsync() {
     let cs = pxt.appTarget.compileService
     return codalGitAsync("checkout", cs.gittag)
         .then(
-        () => /^v\d+/.test(cs.gittag) ? Promise.resolve() : codalGitAsync("pull"),
-        e =>
-            codalGitAsync("checkout", "master")
-                .then(() => codalGitAsync("pull")))
+            () => /^v\d+/.test(cs.gittag) ? Promise.resolve() : codalGitAsync("pull"),
+            e =>
+                codalGitAsync("checkout", "master")
+                    .then(() => codalGitAsync("pull")))
         .then(() => codalGitAsync("checkout", cs.gittag))
 }
 
 // TODO: DAL specific code should be lifted out
-export function buildDalConst(buildEngine: BuildEngine, mainPkg: pxt.MainPackage, force = false) {
+export function buildDalConst(buildEngine: BuildEngine, mainPkg: pxt.MainPackage, rebuild = false,
+    create = false) {
     let constName = "dal.d.ts"
     let vals: Map<string> = {}
     let done: Map<string> = {}
@@ -431,8 +432,8 @@ export function buildDalConst(buildEngine: BuildEngine, mainPkg: pxt.MainPackage
         return outp
     }
 
-    if (mainPkg && (force ||
-        (mainPkg.getFiles().indexOf(constName) >= 0 && !fs.existsSync(constName)))) {
+    if (mainPkg && (create ||
+        (mainPkg.getFiles().indexOf(constName) >= 0 && (rebuild || !fs.existsSync(constName))))) {
         pxt.log(`rebuilding ${constName}...`)
         let files: string[] = []
         let foundConfig = false

cli/cli.ts

@@ -3656,7 +3656,7 @@ function gdbAsync(c: commandParser.ParsedCommand) {
 function buildDalDTSAsync() {
     ensurePkgDir()
     return mainPkg.loadAsync()
-        .then(() => build.buildDalConst(build.thisBuild, mainPkg, true))
+        .then(() => build.buildDalConst(build.thisBuild, mainPkg, true, true))
 }
 
 function buildCoreAsync(buildOpts: BuildCoreOptions): Promise<pxtc.CompileResult> {

localtypings/pxtarget.d.ts

@@ -311,6 +311,7 @@ declare namespace ts.pxtc {
         useMkcd?: boolean;
         useELF?: boolean;
         saveAsPNG?: boolean;
+        noSourceInFlash?: boolean;
         useModulator?: boolean;
         webUSB?: boolean; // use WebUSB when supported
         hexMimeType?: string;

pxtcompiler/emitter/emitter.ts

@@ -3322,6 +3322,7 @@ ${lbl}: .short 0xffff
         }
 
         function emitBrk(node: Node) {
+            bin.numStmts++
             if (!opts.breakpoints)
                 return
             let src = getSourceFileOfNode(node)
@@ -4334,6 +4335,7 @@ ${lbl}: .short 0xffff
         checksumBlock: number[];
         numStmts = 1;
         commSize = 0;
+        packedSource: string;
 
         strings: pxt.Map<string> = {};
         hexlits: pxt.Map<string> = {};
@@ -4348,6 +4350,7 @@ ${lbl}: .short 0xffff
             this.strings = {}
             this.hexlits = {}
             this.doubles = {}
+            this.numStmts = 0
         }
 
         addProc(proc: ir.Procedure) {

pxtcompiler/emitter/hexfile.ts

@@ -509,6 +509,24 @@ namespace ts.pxtc {
                     Util.pushRange(myhex, app)
             }
 
+            if (bin.packedSource) {
+                if (uf2) {
+                    U.userError("TODO")
+                } else {
+                    upper = 0x20000000
+                    addr = 0
+                    myhex.push(hexBytes([0x02, 0x00, 0x00, 0x04, upper >> 8, upper & 0xff]))
+                    for (let i = 0; i < bin.packedSource.length; i += 16) {
+                        let bytes = [0x10, (addr >> 8) & 0xff, addr & 0xff, 0]
+                        for (let j = 0; j < 16; ++j) {
+                            bytes.push((bin.packedSource.charCodeAt(i + j) || 0) & 0xff)
+                        }
+                        myhex.push(hexBytes(bytes))
+                        addr += 16
+                    }
+                }
+            }
+
             if (uf2)
                 return [UF2.serializeFile(uf2)]
             else
@@ -720,46 +738,56 @@ ${hex.hexPrelude()}
         }
     }
 
-    function addSource(meta: string, binstring: string) {
-        let metablob = Util.toUTF8(meta)
-        let totallen = metablob.length + binstring.length
+    function addSource(blob: string) {
+        let res = ""
 
-        if (totallen > 40000) {
-            return "; program too long\n";
+        for (let i = 0; i < blob.length; ++i) {
+            let v = blob.charCodeAt(i) & 0xff
+            if (v <= 0xf)
+                res += "0" + v.toString(16)
+            else
+                res += v.toString(16)
         }
 
-        let str =
-            `
+        return `
     .balign 16
-    .hex 41140E2FB82FA2BB
-    .short ${metablob.length}
-    .short ${binstring.length}
-    .short 0, 0   ; future use
-
-_stored_program: .string "`
-
-        let addblob = (b: string) => {
-            for (let i = 0; i < b.length; ++i) {
-                let v = b.charCodeAt(i) & 0xff
-                if (v <= 0xf)
-                    str += "\\x0" + v.toString(16)
-                else
-                    str += "\\x" + v.toString(16)
-            }
-        }
+_stored_program: .hex ${res}
+`
+    }
+
+    function packSource(meta: string, binstring: string) {
+        let metablob = Util.toUTF8(meta)
+        let totallen = metablob.length + binstring.length
 
-        addblob(metablob)
-        addblob(binstring)
+        let res = "\x41\x14\x0E\x2F\xB8\x2F\xA2\xBB"
 
-        str += "\"\n"
-        return str
+        res += U.uint8ArrayToString([
+            metablob.length & 0xff, metablob.length >> 8,
+            binstring.length & 0xff, binstring.length >> 8,
+            0, 0, 0, 0
+        ])
+
+        res += metablob
+        res += binstring
+
+        if (res.length % 2)
+            res += "\x00"
+
+        return res
     }
 
     export function processorEmit(bin: Binary, opts: CompileOptions, cres: CompileResult) {
         let src = serialize(bin, opts)
         src = patchSrcHash(bin, src)
-        if (opts.embedBlob)
-            src += addSource(opts.embedMeta, ts.pxtc.decodeBase64(opts.embedBlob))
+        let sourceAtTheEnd = false
+        if (opts.embedBlob) {
+            bin.packedSource = packSource(opts.embedMeta, ts.pxtc.decodeBase64(opts.embedBlob))
+            // TODO more dynamic check for source size
+            if (!bin.target.noSourceInFlash && bin.packedSource.length < 40000) {
+                src += addSource(bin.packedSource)
+                bin.packedSource = null // no need to append anymore
+            }
+        }
         let checksumWords = 8
         let pageSize = hex.flashCodeAlign(opts.target)
         if (opts.target.flashChecksumAddr) {
@@ -795,7 +823,6 @@ __flash_checksums:
 `
         }
         bin.writeFile(pxtc.BINARY_ASM, src)
-        bin.numStmts = cres.breakpoints.length
         let res = assemble(opts.target, bin, src)
         if (res.thumbFile.commPtr)
             bin.commSize = res.thumbFile.commPtr - hex.commBase

pxtcompiler/emitter/thumb.ts

@@ -507,6 +507,14 @@ namespace ts.pxtc.thumb {
                 // RULE: beq .next; b .somewhere; .next: -> bne .somewhere
                 ln.update("bne " + lnNext.words[1])
                 lnNext.update("")
+            } else if (lnop == "push" && ln.numArgs[0] == 0x4000 && lnNext.getOp() == "push") {
+                // RULE: push {lr}; push {X, ...} -> push {lr, X, ...}
+                ln.update(lnNext.text.replace("{", "{lr, "))
+                lnNext.update("")
+            } else if (lnop == "pop" && lnNext.getOp() == "pop" && lnNext.numArgs[0] == 0x8000) {
+                // RULE: pop {X, ...}; pop {pc} -> push {X, ..., pc}
+                ln.update(ln.text.replace("}", ", pc}"))
+                lnNext.update("")
             } else if (lnop == "push" && lnNext.getOp() == "pop" && ln.numArgs[0] == lnNext.numArgs[0]) {
                 // RULE: push {X}; pop {X} -> nothing
                 assert(ln.numArgs[0] > 0)

pxtlib/cpp.ts

@@ -445,6 +445,7 @@ namespace pxt.cpp {
                         return "boolean";
                     case "StringData*": return "string";
                     case "String": return "string";
+                    case "ImageLiteral_": return "string";
                     case "ImageLiteral": return "string";
                     case "Action": return "() => void";
 
@@ -479,6 +480,9 @@ namespace pxt.cpp {
                     case "bool": return "B";
                     case "double": return "D"
 
+                    case "ImageLiteral_":
+                        return "T"
+
                     default:
                         if (U.lookup(knownEnums, tp))
                             return "I"

pxtlib/main.ts

@@ -38,6 +38,8 @@ namespace pxt {
         }
         if (!comp.vtableShift)
             comp.vtableShift = 2
+        if (!comp.useUF2 && !comp.useELF && comp.noSourceInFlash == undefined)
+            comp.noSourceInFlash = true // no point putting sources in hex to be flashed
         if (!appTarget.appTheme) appTarget.appTheme = {}
         if (!appTarget.appTheme.embedUrl)
             appTarget.appTheme.embedUrl = appTarget.appTheme.homeUrl

pxtlib/util.ts

@@ -504,7 +504,7 @@ namespace ts.pxtc.Util {
         return res;
     }
 
-    export function uint8ArrayToString(input: Uint8Array) {
+    export function uint8ArrayToString(input: ArrayLike<number>) {
         let len = input.length;
         let res = ""
         for (let i = 0; i < len; ++i)