Remnote2OrgRoam.py

# terminal code: "cd Remnote2Org && python Remnote2OrgRoam.py"

# print("Python execution started")
import sys, os, json, datetime, re

# Import modules from current project:
from progressBar import printProgressBar

# Custom Package installation
from dateutil.parser import parse as dateParse

start_time = datetime.datetime.now()
dir_path = os.path.dirname(os.path.realpath(__file__))

# user-input variables: ----------------------------------------
jsonFile = "../Data/rem.json"
jsonPath = os.path.join(dir_path, jsonFile)
# jsonPath = sys.argv[1]
RemLanguages = "../Data/RemLanguages.json"
langJsonPath = os.path.join(dir_path, RemLanguages)
OrgRootFolder = "Rem2OrgRoam"
dailyDocsFolder = "Daily Documents"
highlightToHTML = False  # if False: Highlights will be '==sampleText==', if True '<mark style=" background-color: {color}; ">{text}</mark>'
previewBlockRef = True
delimiterSR = " -- "  # Spaced Repetition Delimiter

re_HTML = re.compile("(?<!`)<(?!\s|-).+?>(?!`)")
re_newLine = re.compile(
    "(\\n){3,}"
)  # replace more than 2 newlines with only 2: https://regex101.com/r/9VAqaO/1/
re_remID = re.compile(
    r"((\]\])?\s*\[.*?(\]\[))"
)  # reference: https://regex101.com/r/z9B8Pw/2
# ---------------------------------------------------------------
pbr = ""
if previewBlockRef:
    pbr = "!"

if not os.path.isfile(jsonPath):
    sys.exit("JSON file not found")
Rem2ObsPath = os.path.join(dir_path, OrgRootFolder)
os.makedirs(Rem2ObsPath, exist_ok=True)

remnoteJSON = json.load(open(jsonPath, mode="rt", encoding="utf-8", errors="ignore"))
RemnoteDocs = remnoteJSON["docs"]
ignoreKey = ["Remnote Default"]
ignoreID = ["9onq37x6PbsFxvRqu", "6sz2MJeFLZoTRQofZ"]

allParentRem = []
# allFolders = []
# topFolders = []
for x in RemnoteDocs:
    if (
        x.get("n", False) == 1
        and x.get("_id", False) not in ignoreID
        and x["key"] != []
        and x["key"][0] not in ignoreKey
    ):
        allParentRem.append(x)
        if x.get("rcrt", False) == "d":
            # Convert Daily Documents to folder
            x["key"][0] = dailyDocsFolder
            x["forceIsFolder"] = True
    # if "forceIsFolder" in x and  x["forceIsFolder"]:
    #     allFolders.append(x)
    #     if x["parent"] == None:
    #         topFolders.append(x)


def getAllDocs(RemList):
    IDlist = []
    for rem in RemList:
        if rem.get("forceIsFolder", False):
            childRem = []
            for child in rem["children"]:
                dict = [x for x in RemnoteDocs if x["_id"] == child][0]
                childRem.append(dict)
            IDlist.extend(getAllDocs(childRem))
        if (
            len(rem["children"]) > 0
            or (len(rem.get("portalsIn", [])) > 0)
            or (len(rem.get("references", [])) > 0)
            or (len(rem.get("typeChildren", [])) > 0)
        ):
            IDlist.append(rem["_id"])
        else:
            # print("REM not used anywhere")
            pass
    return IDlist


allDocID = getAllDocs(allParentRem)
# allDocID is used in textFromID function

created = []
notCreated = []


def main():
    printProgressBar(
        0, len(allParentRem), prefix="Progress:", suffix="Complete", length=50
    )
    i = 0
    for dict in allParentRem:
        i += 1
        if ignoreRem(dict["_id"]):
            continue
        createFile(dict["_id"], Rem2ObsPath)
        printProgressBar(
            i, len(allParentRem), prefix="Progress:", suffix="Complete", length=50
        )

    timetaken = str(datetime.datetime.now() - start_time)
    print(f"\nTime Taken to Generate '{OrgRootFolder}' Org-Mode Folder: {timetaken}")
    print("\n" + str(len(created)) + " files generated")
    print(
        str(len(notCreated))
        + " file/s listed below could not be generated\n"
        + "\n".join(notCreated)
    ) if len(notCreated) > 0 else None


def createFile(remID, remFolderPath, pathLevel=0):
    # this is recursive function, so cannot be moved directly to main() function
    if ignoreRem(remID):
        return
    remText = textFromID(remID)
    remDict = dictFromID(remID)

    textSplit = remText.split(delimiterSR)
    filename = textSplit[0]
    filename = replaceRemID(filename)
    fileDesc = ""
    if len(textSplit) > 1:
        fileDesc = "\nFile Description: " + textSplit[1]

    if remDict.get("forceIsFolder", False):
        newFilePath = os.path.join(remFolderPath, filename)
        for child in remDict["children"]:
            createFile(child, newFilePath, pathLevel + 1)
    else:
        os.makedirs(remFolderPath, exist_ok=True)
        fileTitle = filename
        # filename = re.sub('[^\w\-_\. ]', '_', filename)
        if os.path.basename(remFolderPath) == dailyDocsFolder:
            # dailyDocName = datetime.datetime.strptime(filename, "%B %dth, %Y").date()
            try:
                dailyDocName = dateParse(filename)
                filename = dailyDocName.strftime("%Y-%m-%d")
            except:
                pass
            # fileTitle += " (" + filename + ")"
        filePath = os.path.join(remFolderPath, filename + ".org")

        try:
            with open(filePath, mode="wt", encoding="utf-8") as f:
                child = expandChildren(remID, pathLevel=pathLevel)
                fileMetadata = f":PROPERTIES:\n:ID:       {remID}\n:END:\n#+title: "
                # if child == []:
                #     # if there are not children, do not generate file (could cause issues with REM that are referenced without any actual content)
                #     raise ValueError(filename + '.org File doesnt have any content')
                expandBullets = "\n".join(child)

                f.write(fileMetadata + fileTitle + fileDesc + "\n\n" + expandBullets)
            # print(f'{filename}.org created')
            created.append("ID: " + remID + ",  Name: " + filename)
        except Exception as e:
            # print(e)
            notCreated.append("ID: " + remID + ",  Name: " + filename)
            # print("\ncannot create file with ID: " + remID + ", Name: "+ filename + "\n")


def ignoreRem(ID):
    # TODO: add more ignore ID's
    ignoreID = ["9onq37x6PbsFxvRqu", "6sz2MJeFLZoTRQofZ"]
    dict = dictFromID(ID)
    if (
        dict == []
        or dict["key"] == []
        or ("contains:" in dict["key"])
        or ("rcrp" in dict)
        or ("rcrs" in dict)
        or ("rcrt" in dict and dict.get("rcrt") != "c" and dict.get("rcrt") != "d")
        or (dict.get("type", False) == 6)
    ):
        return True
    else:
        return False


def expandChildren(ID, level=0, pathLevel=0):
    childIDList = [x["children"] for x in RemnoteDocs if x["_id"] == ID][0]
    filteredChildren = []
    text = ""
    childData = [x for x in RemnoteDocs if x["_id"] in childIDList]
    for x in childData:
        ChildID = x["_id"]
        if not ignoreRem(ChildID):
            text = textFromID(ChildID, pathLevel=pathLevel)
            prefix = ""
            if level >= 1:
                prefix = "*" * level
            prefix += "* "
            blankPrefix = prefix.replace("*", " ")
            # if text.startswith("#+BEGIN_SRC"): # not necessary - this is removing bullet in first line of code-block
            #     prefix = blankPrefix
            text = prefix + text
            if "references" in x and x["references"] != []:
                orgRoamData = ["", ":PROPERTIES:", f":ID:       {ChildID}", ":END:", ""]
                text += f"\n".join(orgRoamData)
            if "\n" in text:
                text = text.replace("\r", "\n")
                text = re.sub(re_newLine, r"\n\n", text)
                text = text.replace("\n", "\n" + blankPrefix)
            filteredChildren.append(text)

            filteredChildren.extend(expandChildren(ChildID, level=level + 1))

    return filteredChildren


def dictFromID(ID):
    dict = []
    try:
        dict = [x for x in RemnoteDocs if x["_id"] == ID][0]
    except Exception as e:
        # print(e)
        # print(f"REM with ID: '{ID}' not found")
        pass
    return dict


def textFromID(ID, level=0, pathLevel=0):
    dict = dictFromID(ID)
    key = dict["key"]
    text = ""

    todoStatus = getTODO(dict)
    if todoStatus == "Finished":
        text += "DONE "
    elif todoStatus == "Unfinished":
        text += "TODO "

    text += arrayToText(key, ID, pathLevel=pathLevel)

    # value = dict.get("value", [])
    # if value and len(value) > 0:
    #     text += delimiterSR + arrayToText(value, ID, pathLevel = pathLevel)

    if level == 0:
        # level is used to disable recursive expansion, since tags don't need to be recursive
        if (
            (len(dict.get("typeParents", [])) > 0)
            and not ID in allDocID
            and not (dict.get("forceIsFolder", False))
        ):
            text += convertTags(dict)

    if text.startswith("#+BEGIN_SRC"):
        text = text.replace("\r\n", "\n")
        # in Windows - "\r\n" means end of line - https://stackoverflow.com/a/1761086/6908282

    return text


def arrayToText(array, ID, pathLevel=0):
    text = ""
    for item in array:
        if isinstance(item, str):
            text += fence_HTMLtags(item)
        elif item["i"] == "q" and "_id" in item:
            newDict = dictFromID(item["_id"])
            if newDict == []:
                continue
            newID = newDict["_id"]
            # TODO parentPath needs to be corrected - for paths in same parent folder, this still adds all folders
            parentPath = newID  # parentFromID(newID)
            IDtext = textFromID(newID)
            IDtext = replaceRemID(IDtext)
            # Reference: https://regex101.com/r/z9B8Pw/1
            # \s*\\\[.*\[|\\]\\] - keeps the tag text in the reference
            # \s*\\\[.*\\]\\] - remove the entire tag from reference
            refPrefix = "id:"  # + ("../"*pathLevel)
            if newID in allDocID:
                text += f"[[{refPrefix}{parentPath}][{IDtext}]]"
            else:
                # TODO Org-Tansclution: https://org-roam.discourse.group/t/alpha-org-transclusion/830
                text += f"[[{refPrefix}{parentPath}][{IDtext}]]"
        elif item["i"] == "o":
            text += f'#+BEGIN_SRC {getOrgLanguage(item.get("language", "None"))}\n{item["text"]}\n#+END_SRC'
        elif item["i"] == "i" and "url" in item:
            text += f'[[{item["url"]}]]'
        elif item["i"] == "m":
            currText = item["text"]
            currText = fence_HTMLtags(currText)
            if "url" in item:
                text += f'[[{item["url"]}][{currText.strip()}]]'
            elif currText.strip() == "":
                text += currText
            elif item.get("q", False):
                text += f"~{currText}~"
            elif item.get("x", False):
                text += f"$${currText}$$"
            elif item.get("b", False):
                if item.get("h", False):
                    text = textHighlight(currText, item["h"], html=highlightToHTML)
                else:
                    text += f"*{currText}*"
            elif item.get("h", False):
                text += textHighlight(currText, item["h"], html=highlightToHTML)
            elif item.get("u", False):
                text += currText
        elif item["i"] == "q" and "textOfDeletedRem" in item:
            text += "#DeletedRem: " + "".join(item["textOfDeletedRem"])
        else:
            print("Could not Extract text at textFromID function for ID: " + ID)

    return text


def replaceRemID(text):
    text = re.sub(re_remID, " #", text)
    text = text.replace("]]", "")
    text = text.replace("/", "")  # replace "/" if added in parentFromID() function
    text = text.strip()

    return text


def convertTags(dict):
    text = ""
    for id in dict["typeParents"]:
        if not ignoreRem(id):
            textExtract = textFromID(id, level=1).strip()
            textExtract = re.sub(r"[^A-Za-z0-9-]+", "_", textExtract)
            text += f" [[id:{id}][{textExtract}]]"

    return text


def textHighlight(text, colorNum, html=False):
    if html:
        # Switch-Case: https://stackoverflow.com/a/60211/6908282
        def switch(x):
            colorList = {
                1: "firebrick",
                2: "darkorange",
                3: "goldenrod",
                4: "seagreen",
                5: "rebeccapurple",
                6: "steelblue",
            }

            color = colorList.get(x, "")
            return color

        color = switch(colorNum)
        text = f'<mark style=" background-color: {color}; ">{text}</mark>'
    else:
        text = f"=={text}=="

    return text


def getTODO(keyDict):
    if isinstance(keyDict["key"][0], dict) and keyDict["key"][0].get("i") == "o":
        # Excludue "Custom CSS" Rem - Dont add Todo check-boxes from here
        # Typically, CSS code-block has only one item in the key dictionary and all code-block have property `"i": "o"`
        return False
    todo = keyDict.get("crt")
    if todo and "t" in todo:
        todoStatus = todo["t"]["s"]["s"]
        return todoStatus
    else:
        return False


def fence_HTMLtags(string):
    # Reference: https://regex101.com/r/BVWwGK/10
    if not string.startswith("```"):
        # \g<0> stands for whole match - so we're adding backtick (`) as suffix and prefix for whole match
        # reference: https://docs.python.org/3/library/re.html#re.sub
        # \g<0> instead of \0 - reference: https://stackoverflow.com/q/58134893/6908282
        string = re.sub(re_HTML, r"`\g<0>`", string)
    return string


def parentFromID(ID):
    fileName = ""
    dict = dictFromID(ID)
    if ID in allDocID:
        filePath = getFilePath(ID)
        filePath.reverse()
        fileName = "/".join(filePath) + "/" + textFromID(ID)
    else:
        fileName = parentFromID(dict["parent"])

    return fileName


def getFilePath(ID):
    pathList = []
    dict = dictFromID(ID)
    if dict != [] and dict.get("parent", None) != None:
        pathList.append(textFromID(dict["parent"]))
        pathList.extend(getFilePath(dict["parent"]))

    return pathList


def getOrgLanguage(lang):
    lang = lang.lower()
    langList = json.load(
        open(langJsonPath, mode="rt", encoding="utf-8", errors="ignore")
    )
    try:
        identifier = langList[lang]
    except Exception as e:
        identifier = langList[lang]
        print(e)
        print("cannot find org-language(syntax-highlight) for: " + lang)

    return identifier


if __name__ == "__main__":
    main()