Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP

Loading…

Debian Packages #32

Closed
wants to merge 3 commits into from

1 participant

@mrshu
Owner

This pull request adds Debian Packages as fathead.

Please note that it merges packages from stable, unsable and testing.

@nospampleasemam nospampleasemam referenced this pull request in duckduckgo/zeroclickinfo-spice
Closed

Deb version #101

@mrshu
Owner

To clean things up, I'm closing this.

Is that OK with you @nospampleasemam?

I believe this pull request is now just a piece of history =)

@mrshu mrshu closed this
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Commits on Jul 12, 2012
  1. @mrshu

    added debian packages fathead

    mrshu authored
  2. @mrshu

    updated README

    mrshu authored
Commits on Jul 28, 2012
  1. @mrshu
This page is out of date. Refresh to see the latest.
View
6 debian_pkgs/README.txt
@@ -0,0 +1,6 @@
+Debian Packages plugin for DukcDuckGo
+
+Dependencies:
+
+Python 2.7
+gzip (should be included in Python)
View
7 debian_pkgs/fetch.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+mkdir -p download
+
+wget -q -N 'http://packages.debian.org/unstable/allpackages?format=txt.gz' -O 'download/unstable.txt.gz'
+wget -q -N 'http://packages.debian.org/stable/allpackages?format=txt.gz' -O 'download/stable.txt.gz'
+wget -q -N 'http://packages.debian.org/testing/allpackages?format=txt.gz' -O 'download/testing.txt.gz'
+
View
15 debian_pkgs/meta.txt
@@ -0,0 +1,15 @@
+# This is the name of the source as people would refer to it, e.g. Wikipedia or PerlDoc
+Name: Debian Packages
+
+# This is the base domain where the source pages are located.
+Domain: packages.debian.org
+
+# This is what gets put in quotes next to the source
+# It can be blank if it is a source with completely general info spanning many types of topics like Facebook.
+Type: Debian Package
+
+# Whether the source is from MediaWiki (1) or not (0).
+MediaWiki: 0
+
+# Keywords uses to trigger (or prefer) the source over others.
+Keywords: debian package,deb
View
112 debian_pkgs/parse.py
@@ -0,0 +1,112 @@
+# -*- coding: utf-8 -*-
+
+import logging
+import os
+import gzip
+import re
+import string
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger()
+
+class Package(object):
+ """ Contains informations about an Ubuntu package"""
+ def __init__(self, name, info, reference):
+ self.name = name
+ self.info = info
+ self.reference = reference
+
+ def __str__(self):
+ fields = [
+ self.name, # $page
+ '', # $namespace
+ self.reference, # $url
+ self.info, # $description
+ '', # $synopsis (code)
+ '', # $details
+ 'A', # $type
+ '' # $lang
+ ]
+
+ output = '%s' % ('\t'.join(fields))
+
+ return output
+
+
+class Parser(object):
+ """ Parses a HTML file to get
+ all packages from it"""
+
+ UBUNTU_PKGS_URL = 'http://packages.debian.org'
+
+ def __init__(self, input='download/stable.txt.gz'):
+ self.input = gzip.open(input, 'rb')
+
+ for x in range(6):
+ self.input.readline()
+
+ def get_packages(self):
+ """ """
+
+ self.packages = []
+ for line in self.input:
+
+ if '(' in line:
+ data = re.match('(.*?) \(.*?\) (.*)', line).groups()
+ name = data[0]
+ info = data[1]
+ else:
+ data = line.split(' ')
+ name = data[0]
+ info = ' '.join(data[1::])
+
+ # fix for agda-bin package; removing non-ascii characters
+ info = filter(lambda x: x in string.printable, info)
+
+ if '[' in info:
+ data = re.match('\[(.*?)\] (.*)', info)
+ if data:
+ data = data.groups()
+ info = data[1] + ' [' + data[0] + ']'
+ else:
+ info = re.sub('\[(.*?)\]', '', info)
+
+ info = info.rstrip('\n')
+
+ reference = self.UBUNTU_PKGS_URL + '/' + name
+
+ package = Package(name, info, reference)
+ self.packages.append(package)
+
+ logger.info('Parsed package %s' % name)
+
+if __name__ == '__main__':
+ stable = Parser()
+ stable.get_packages()
+
+ unstable = Parser('download/unstable.txt.gz')
+ unstable.get_packages()
+
+ testing = Parser('download/testing.txt.gz')
+ testing.get_packages()
+
+ appended = []
+ with open('output.txt', 'w') as output:
+ for package in stable.packages:
+ output.write(package.__str__().encode('utf-8') + '\n')
+ logger.info('Package added to output: %s' % package.name)
+ appended.append(package.name)
+
+ for package in unstable.packages:
+ if package.name not in appended:
+ output.write(package.__str__().encode('utf-8') + '\n')
+ logger.info('Package added to output: %s' % package.name)
+ appended.append(package.name)
+
+ for package in testing.packages:
+ if package.name not in appended:
+ output.write(package.__str__().encode('utf-8') + '\n')
+ logger.info('Package added to output: %s' % package.name)
+ appended.append(package.name)
+
+
View
2  debian_pkgs/parse.sh
@@ -0,0 +1,2 @@
+#!/bin/bash
+python parse.py
View
0  debian_pkgs/queries.txt
No changes.
Something went wrong with that request. Please try again.