Skip to content
This repository has been archived by the owner on Aug 9, 2021. It is now read-only.

Commit

Permalink
feat(package): defer apk / upk parsing (#127)
Browse files Browse the repository at this point in the history 
* refactor(schema): remove useless collation in create table queries
* feat(package): defer apk / upk parsing
* refactor(upload): defer parse of applications
  • Loading branch information
@btry @ajsb85
btry authored and ajsb85 committed Oct 3, 2017
1 parent 4cb8eeb commit c9e9ed6
Show file tree
Hide file tree
Showing 7 changed files with 150 additions and 89 deletions.
169 changes: 109 additions & 60 deletions inc/package.class.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -232,29 +232,6 @@ public function prepareInputForAdd($input) {
$this->createEntityDirectory(dirname($destination));
if (rename($uploadedFile, $destination)) {
$fileExtension = pathinfo($destination, PATHINFO_EXTENSION);
$filename = pathinfo($destination, PATHINFO_FILENAME);
if ($fileExtension == 'apk') {
$apk = new \ApkParser\Parser($destination);
} else if ($fileExtension == 'upk') {
$upkParser = new PluginFlyvemdmUpkparser($destination);
$apk = $upkParser->getApkParser();
if (!($apk instanceof \ApkParser\Parser)) {
Session::addMessageAfterRedirect(__('Could not parse the UPK file', 'flyvemdm'));
return false;
}
}
$manifest = $apk->getManifest();
$iconResourceId = $manifest->getApplication()->getIcon();
$labelResourceId = $manifest->getApplication()->getLabel();
$iconResources = $apk->getResources($iconResourceId);
$apkLabel = $apk->getResources($labelResourceId);
$input['icon'] = base64_encode(stream_get_contents($apk->getStream($iconResources[0])));
$input['name'] = $manifest->getPackageName();
if ((!isset($input['alias'])) || (strlen($input['alias']) == 0)) {
$input['alias'] = $apkLabel[0]; // Get the first item
}
$input['version'] = $manifest->getVersionName();
$input['version_code'] = $manifest->getVersionCode();
$input['filesize'] = fileSize($destination);
$input['dl_filename'] = basename($uploadedFile);
} else {
Expand Down Expand Up @@ -317,48 +294,30 @@ public function prepareInputForUpdate($input) {
}
}

try {
$input['filename'] = $this->fields['entities_id'] . "/" . uniqid() . "_" . basename($uploadedFile);
$destination = FLYVEMDM_PACKAGE_PATH . "/" . $input['filename'];
$this->createEntityDirectory(dirname($destination));
if (rename($uploadedFile, $destination)) {
if ($fileExtension == "apk") {
$apk = new \ApkParser\Parser($destination);
} else if ($fileExtension == "upk") {
$upkParser = new PluginFlyvemdmUpkparser($destination);
$apk = $upkParser->getApkParser();
if (!($apk instanceof \ApkParser\Parser)) {
Session::addMessageAfterRedirect(__('Could not parse the UPK file', "flyvemdm"));
return false;
if (isset($uploadedFile)) {
try {
$input['filename'] = $this->fields['entities_id'] . "/" . uniqid() . "_" . basename($uploadedFile);
$destination = FLYVEMDM_PACKAGE_PATH . "/" . $input['filename'];
$this->createEntityDirectory(dirname($destination));
if (rename($uploadedFile, $destination)) {
$input['filesize'] = fileSize($destination);
$input['dl_filename'] = basename($uploadedFile);
if ($filename != $this->fields['filename']) {
unlink(FLYVEMDM_PACKAGE_PATH . "/" . $this->fields['filename']);
}
} else {
if (!is_writable(dirname($destination))) {
$destination = dirname($destination);
Toolbox::logInFile('php-errors', "Plugin Flyvemdm : Directory '$destination' is not writeable");
}
Session::addMessageAfterRedirect(__('Unable to save the file', "flyvemdm"));
$input = false;
}
$manifest = $apk->getManifest();
$iconResourceId = $manifest->getApplication()->getIcon();
$labelResourceId = $manifest->getApplication()->getLabel();
$iconResources = $apk->getResources($iconResourceId);
$apkLabel = $apk->getResources($labelResourceId);
$input['icon'] = base64_encode(stream_get_contents($apk->getStream($iconResources[0])));
$input['name'] = $manifest->getPackageName();
$input['version'] = $manifest->getVersionName();
$input['version_code'] = $manifest->getVersionCode();
$input['filesize'] = fileSize($destination);
$input['dl_filename'] = basename($uploadedFile);
if ($filename != $this->fields['filename']) {
unlink(FLYVEMDM_PACKAGE_PATH . "/" . $this->fields['filename']);
}
} else {
if (!is_writable(dirname($destination))) {
$destination = dirname($destination);
Toolbox::logInFile('php-errors', "Plugin Flyvemdm : Directory '$destination' is not writeable");
}
Session::addMessageAfterRedirect(__('Unable to save the file', "flyvemdm"));
} catch (Exception $e) {
// Ignore exceptions for now
$input = false;
}
} catch (Exception $e) {
// Ignore exceptions for now
$input = false;
}

return $input;
}

Expand Down Expand Up @@ -594,6 +553,96 @@ protected function sendFile() {
exit(0);
}

/**
* get Cron description parameter for this class
*
* @param $name string name of the task
*
* @return array of string
**/
static function cronInfo($name) {

switch ($name) {
case 'ParseApplication' :
return array('description' => __('Parse an application to find metadata', 'flyvemdm'));
}
}


/**
* Launches parsing of applciation files
*
* @see PluginFlyvemdmPackage::parseApplication()
*
* @param CronTask $crontask
*
* @return integer >0 means done, < 0 means not finished, 0 means nothing to do
*/
public static function cronParseApplication(CronTask $crontask) {
global $DB;

$cronStatus = 0;

$request = [
'FROM' => static::getTable(),
'WHERE' => ['AND' => [
'parse_status' => 'pending',
]],
'LIMIT' => 10
];
foreach ($DB->request($request) as $data) {
$package = new static();
$package->getFromDB($data['id']);
if ($package->parseApplication()) {
$cronStatus++;
}
}

return $cronStatus;
}

/**
* Analyzes an application (APK or UPK) to collect metadata
*
* @return boolean true if success, false otherwise
*/
private function parseApplication() {
$destination = FLYVEMDM_PACKAGE_PATH . '/' . $this->fields['filename'];
$fileExtension = pathinfo($destination, PATHINFO_EXTENSION);
if ($fileExtension == 'apk') {
$apk = new \ApkParser\Parser($destination);
} else if ($fileExtension == 'upk') {
$upkParser = new PluginFlyvemdmUpkparser($destination);
$apk = $upkParser->getApkParser();
if (!($apk instanceof \ApkParser\Parser)) {
$this->update([
'id' => $this->fields['id'],
'parse_status' => 'failed'
]);
return false;
}
}
$manifest = $apk->getManifest();
$iconResourceId = $manifest->getApplication()->getIcon();
$labelResourceId = $manifest->getApplication()->getLabel();
$iconResources = $apk->getResources($iconResourceId);
$apkLabel = $apk->getResources($labelResourceId);

$input = [];
$input['icon'] = base64_encode(stream_get_contents($apk->getStream($iconResources[0])));
$input['name'] = $manifest->getPackageName();
if ((!isset($input['alias'])) || (strlen($input['alias']) == 0)) {
$input['alias'] = $apkLabel[0]; // Get the first item
}
$input['version'] = $manifest->getVersionName();
$input['version_code'] = $manifest->getVersionCode();

$input['id'] = $this->fields['id'];
$input['parse_status'] = 'parsed';
$this->update($input);
return true;
}

/**
* Deletes the packages related to the entity
* @param CommonDBTM $item
Expand Down
15 changes: 10 additions & 5 deletions install/installer.class.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -490,7 +490,6 @@ protected function upgrade($fromVersion) {
* @param string $toVersion
*/
protected function upgradeOneStep($toVersion) {

$suffix = str_replace('.', '_', $toVersion);
$includeFile = __DIR__ . "/upgrade/update_to_$suffix.php";
if (is_readable($includeFile) && is_file($includeFile)) {
Expand All @@ -507,10 +506,16 @@ protected function upgradeOneStep($toVersion) {

protected function createJobs() {
CronTask::Register(PluginFlyvemdmMqttupdatequeue::class, 'UpdateTopics', MINUTE_TIMESTAMP,
[
'comment' => __('Update retained MQTT topics for fleet policies', 'flyvemdm'),
'mode' => CronTask::MODE_EXTERNAL
]);
[
'comment' => __('Update retained MQTT topics for fleet policies', 'flyvemdm'),
'mode' => CronTask::MODE_EXTERNAL
]);

CronTask::Register(PluginFlyvemdmPackage::class, 'ParseApplication', MINUTE_TIMESTAMP,
[
'comment' => __('Parse uploaded applications to collect metadata', 'flyvemdm'),
'mode' => CronTask::MODE_EXTERNAL
]);
}

/**
Expand Down
45 changes: 23 additions & 22 deletions install/mysql/plugin_flyvemdm_empty.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,8 +46,8 @@ CREATE TABLE IF NOT EXISTS `glpi_plugin_flyvemdm_entityconfigs` (
DROP TABLE IF EXISTS `glpi_plugin_flyvemdm_files`;
CREATE TABLE IF NOT EXISTS `glpi_plugin_flyvemdm_files` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`name` varchar(255) COLLATE utf8_unicode_ci NOT NULL DEFAULT '',
`source` varchar(255) COLLATE utf8_unicode_ci NOT NULL DEFAULT '',
`name` varchar(255) NOT NULL DEFAULT '',
`source` varchar(255) NOT NULL DEFAULT '',
`entities_id` int(11) NOT NULL DEFAULT '0',
`version` int(11) NOT NULL DEFAULT '0',
PRIMARY KEY (`id`)
Expand All @@ -58,7 +58,7 @@ CREATE TABLE IF NOT EXISTS `glpi_plugin_flyvemdm_files` (
DROP TABLE IF EXISTS `glpi_plugin_flyvemdm_fleets`;
CREATE TABLE IF NOT EXISTS `glpi_plugin_flyvemdm_fleets` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`name` varchar(255) COLLATE utf8_unicode_ci NOT NULL DEFAULT '',
`name` varchar(255) NOT NULL DEFAULT '',
`entities_id` int(11) NOT NULL DEFAULT '0',
`is_recursive` int(1) NOT NULL DEFAULT '0',
`is_default` int(1) NOT NULL DEFAULT '0',
Expand All @@ -71,8 +71,8 @@ DROP TABLE IF EXISTS `glpi_plugin_flyvemdm_geolocations`;
CREATE TABLE IF NOT EXISTS `glpi_plugin_flyvemdm_geolocations` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`computers_id` int(11) NOT NULL DEFAULT '0',
`latitude` varchar(255) COLLATE utf8_unicode_ci NOT NULL DEFAULT '',
`longitude` varchar(255) COLLATE utf8_unicode_ci NOT NULL DEFAULT '',
`latitude` varchar(255) NOT NULL DEFAULT '',
`longitude` varchar(255) NOT NULL DEFAULT '',
`date` datetime NOT NULL DEFAULT '0000-00-00 00:00:00',
PRIMARY KEY (`id`)
) ENGINE=MyISAM DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
Expand All @@ -83,7 +83,7 @@ DROP TABLE IF EXISTS `glpi_plugin_flyvemdm_mqttacls`;
CREATE TABLE IF NOT EXISTS `glpi_plugin_flyvemdm_mqttacls` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`plugin_flyvemdm_mqttusers_id` int(11) NOT NULL DEFAULT '0',
`topic` varchar(255) COLLATE utf8_unicode_ci NOT NULL DEFAULT '',
`topic` varchar(255) NOT NULL DEFAULT '',
`access_level` int(11) NOT NULL DEFAULT '0',
PRIMARY KEY (`id`),
UNIQUE KEY `unicity` (`plugin_flyvemdm_mqttusers_id`,`topic`)
Expand All @@ -95,9 +95,9 @@ DROP TABLE IF EXISTS `glpi_plugin_flyvemdm_mqttlogs`;
CREATE TABLE IF NOT EXISTS `glpi_plugin_flyvemdm_mqttlogs` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`date` datetime DEFAULT NULL,
`direction` varchar(1) COLLATE utf8_unicode_ci NOT NULL DEFAULT '' COMMENT 'I for received message, O for sent message',
`topic` varchar(255) COLLATE utf8_unicode_ci NOT NULL DEFAULT '',
`message` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`direction` varchar(1) NOT NULL DEFAULT '' COMMENT 'I for received message, O for sent message',
`topic` varchar(255) NOT NULL DEFAULT '',
`message` varchar(255) DEFAULT NULL,
PRIMARY KEY (`id`),
KEY `date` (`date`),
KEY `topic` (`topic`)
Expand All @@ -108,8 +108,8 @@ CREATE TABLE IF NOT EXISTS `glpi_plugin_flyvemdm_mqttlogs` (
DROP TABLE IF EXISTS `glpi_plugin_flyvemdm_mqttusers`;
CREATE TABLE IF NOT EXISTS `glpi_plugin_flyvemdm_mqttusers` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`user` varchar(255) COLLATE utf8_unicode_ci NOT NULL DEFAULT '',
`password` varchar(255) COLLATE utf8_unicode_ci NOT NULL DEFAULT '',
`user` varchar(255) NOT NULL DEFAULT '',
`password` varchar(255) NOT NULL DEFAULT '',
`enabled` tinyint(1) NOT NULL DEFAULT '0',
PRIMARY KEY (`id`),
UNIQUE KEY `unicity` (`user`)
Expand All @@ -119,16 +119,17 @@ CREATE TABLE IF NOT EXISTS `glpi_plugin_flyvemdm_mqttusers` (
-- Export de la structure de table glpi-flyvemdm. glpi_plugin_flyvemdm_packages
DROP TABLE IF EXISTS `glpi_plugin_flyvemdm_packages`;
CREATE TABLE IF NOT EXISTS `glpi_plugin_flyvemdm_packages` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`name` varchar(255) COLLATE utf8_unicode_ci NOT NULL DEFAULT '',
`alias` varchar(255) COLLATE utf8_unicode_ci NOT NULL DEFAULT '',
`version` varchar(255) COLLATE utf8_unicode_ci NOT NULL DEFAULT '',
`version_code` varchar(255) COLLATE utf8_unicode_ci NOT NULL DEFAULT '',
`icon` text COLLATE utf8_unicode_ci NOT NULL,
`filename` varchar(255) COLLATE utf8_unicode_ci NOT NULL DEFAULT '',
`filesize` int(11) NOT NULL DEFAULT '0',
`entities_id` int(11) NOT NULL DEFAULT '0',
`dl_filename` varchar(255) COLLATE utf8_unicode_ci NOT NULL DEFAULT '',
`id` int(11) NOT NULL AUTO_INCREMENT,
`name` varchar(255) NOT NULL DEFAULT '',
`alias` varchar(255) NOT NULL DEFAULT '',
`version` varchar(255) NOT NULL DEFAULT '',
`version_code` varchar(255) NOT NULL DEFAULT '',
`icon` text COLLATE utf8_unicode_ci NOT NULL,
`filename` varchar(255) NOT NULL DEFAULT '',
`filesize` int(11) NOT NULL DEFAULT '0',
`entities_id` int(11) NOT NULL DEFAULT '0',
`dl_filename` varchar(255) NOT NULL DEFAULT '',
`parse_status` enum('pending', 'parsed', 'failed') NOT NULL DEFAULT 'pending',
PRIMARY KEY (`id`)
) ENGINE=MyISAM DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;

Expand All @@ -137,7 +138,7 @@ CREATE TABLE IF NOT EXISTS `glpi_plugin_flyvemdm_packages` (
DROP TABLE IF EXISTS `glpi_plugin_flyvemdm_wellknownpaths`;
CREATE TABLE IF NOT EXISTS `glpi_plugin_flyvemdm_wellknownpaths` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`name` varchar(255) COLLATE utf8_unicode_ci NOT NULL,
`name` varchar(255) NOT NULL,
`comment` text COLLATE utf8_unicode_ci NOT NULL,
`is_default` tinyint(1) NOT NULL DEFAULT '0',
PRIMARY KEY (`id`),
Expand Down
4 changes: 4 additions & 0 deletions install/upgrade/update_to_dev.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,4 +102,8 @@ function plugin_flyvemdm_update_to_dev(Migration $migration) {
// All policies exist for Android
$migration->addPostQuery("UPDATE `$table` SET `is_android_policy` = '1'");

// update Applications table
$table = 'glpi_plugin_flyvemdm_packages';
$migration->addField($table, 'parse_status', "enum('pending', 'parsed', 'failed')", ['after' => 'dl_filename', 'default' => 'pending']);
$migration->addPostQuery("UPDATE `$table` SET `parse_status` = 'parsed'");
}
4 changes: 3 additions & 1 deletion setup.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,9 @@ function plugin_init_flyvemdm() {

if ($plugin->isActivated('flyvemdm')) {
require_once(__DIR__ . '/vendor/autoload.php');
require_once(__DIR__ . '/lib/GlpiLocalesExtension.php');
if (!class_exists('GlpiLocalesExtension')) {
require_once(__DIR__ . '/lib/GlpiLocalesExtension.php');
}

plugin_flyvemdm_registerClasses();
plugin_flyvemdm_addHooks();
Expand Down
2 changes: 1 addition & 1 deletion tests/suite-install/Config.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ class Config extends CommonTestCase
{

/**
*
* @engine inline
*/
public function testInstallPlugin() {
global $DB;
Expand Down
Empty file modified tools/cli_install.php
100755 → 100644
View file
Empty file.

0 comments on commit c9e9ed6

Please sign in to comment.