add crx2ff as proof that it works but gonna remove it later

mdamien · May 14, 2016 · 89a4c2b · 89a4c2b
1 parent 683ce16
commit 89a4c2b
Show file tree

Hide file tree

Showing 6 changed files with 36 additions and 22 deletions.
diff --git a/TODO b/TODO
@@ -2,12 +2,11 @@
 
 	fix "not deleted but not in final list" problem
 	   => mark as deleted if not in sitemap anymore ?
-	sitemap should be reference for crawl_crx, making the process only
-	    [CRAWL_SITEMAP => CRAWL_CRX]
+	rename .zip to .crx
 	crx4chrome
 
 - site -
 
+	search + filtering for ext metadata + sourcecode
 	diff
 	make data available as json
-	crx2ff
diff --git a/crawl_crx.py b/crawl_crx.py
@@ -41,10 +41,11 @@
     extlist = list(reversed(extlist))
 """
 
-def bad(x): colored(x, 'red')
-def good(x): colored(x, 'green')
+def bad(x): return colored(x, 'red')
+def good(x): return colored(x, 'green')
 
-#print = lambda *x: ''
+ok = print
+print = lambda *x: ''
 
 #@deco.concurrent
 def do(url):
@@ -112,7 +113,7 @@ def do(url):
                     'and .version=', manifest.get('version'))
                 os.remove(tmp_file)
                 return
-            print(good('version is added :D'))
+            ok(good('version is added :D'), manifest['version'], url)
             # assert current_version == version_name or version
             os.makedirs(target_dir_path, exist_ok=True)
             shutil.move(tmp_file, target_file_path)

diff --git a/crx_stats.py b/crx_stats.py
@@ -20,16 +20,20 @@ def keyfunc(filename):
 for ext in tqdm(os.listdir(DIR)):
     files = os.listdir(DIR+ext)
     files_details = []
-    for file in sort_semverfiles(files):
-        fullpath = DIR+ext+'/'+file
-        size = os.path.getsize(fullpath)
-        if size < 10:
-            print(ext, file, 'IS 0000000', size)
-            TO_RM.append('rm '+fullpath)
-        files_details.append({
-            'name': file,
-            'size': size,
-        })
+    try:
+        for file in sort_semverfiles(files):
+            fullpath = DIR+ext+'/'+file
+            size = os.path.getsize(fullpath)
+            if size < 10:
+                print(ext, file, 'IS 0000000', size)
+                TO_RM.append('rm '+fullpath)
+            files_details.append({
+                'name': file,
+                'size': size,
+            })
+    except TypeError as e:
+        print('error with ', ext, files)
+        raise e
 
     exts.append({
         'ext': ext,

diff --git a/extstats/parse_infos.py b/extstats/parse_infos.py
@@ -83,7 +83,7 @@ def extract_manifest_of_file(crx_file):
     if os.path.isfile(crx_file):
         size = os.path.getsize(crx_file)
         if size == 0:
-            print('###################!!!! fucking empty downloads')
+            print('###################!!!! fucking empty downloads', crx_file)
             return
         try:
             with ZipFile(crx_file) as myzip:

diff --git a/extstats/template.html b/extstats/template.html
@@ -33,13 +33,19 @@ <h2 id="{{ ext.ext_id }}">
             <a href="{{ ext.url }}">{{ ext.name }}</a>
         </h2>
         <small>{{ ext.user_count | add_commas }} users</small>
+        <small class='crx2ff-links'>
+                    <a target="_blank" rel="noreferrer" href="{{ CRX2FF_URL }}?id={{ ext.ext_id }}">convert to firefox ext</a> /
+                    <a target="_blank" rel="noreferrer" href="{{ CRX2FF_URL }}?id={{ ext.ext_id }}&check">check firefox compat</a>
+        </small>
         <ul>
         {% for file in ext.files %}
             <li>
                 <a href="{{ file.storage_url }}">
                     {{ file.name.replace('.zip','') }} - <small>{{ file.size | filesizeformat }}</small>
                 </a>
-                <small><a target="_blank" rel="noreferrer" href="{{ VIEW_SOURCE_URL }}{{ file.storage_url }}">(view source)</a></small>
+                <small>
+                    <a target="_blank" rel="noreferrer" href="{{ VIEW_SOURCE_URL }}{{ file.storage_url }}">view source</a>
+                </small>
             </li>
         {% endfor %}
         </ul>

diff --git a/make_site.py b/make_site.py
@@ -12,7 +12,7 @@ def add_commas(eval_ctx, value):
 env.filters['add_commas'] = add_commas
 template = env.get_template('template.html')
 
-
+CRX2FF_URL = "https://crx2ff-yfrezangwq.now.sh"
 VIEW_SOURCE_URL = "https://robwu.nl/crxviewer/crxviewer.html?crx="
 
 exts = json.load(open('data/PAGES.json'))
@@ -52,7 +52,11 @@ def split_list(L, n):
 for i, group in enumerate(exts_groups):
 	page = i + 1
 	result = template.render(exts=group, exts_count=len(exts),
-	    files_count=files_count, VIEW_SOURCE_URL=VIEW_SOURCE_URL,
-	    now=datetime.datetime.now(), pages=len(exts_groups), page=page)
+	    files_count=files_count,
+        VIEW_SOURCE_URL=VIEW_SOURCE_URL,
+        CRX2FF_URL=CRX2FF_URL,
+	    now=datetime.datetime.now(),
+        pages=len(exts_groups),
+        page=page)
 	name = 'pages/' + str(page) if page > 1 else 'index'
 	open('../site/chrome-extensions-archive/{}.html'.format(name), 'w').write(result)