From 37b3704f82d65e54e37e440e770bf1eb04bec5ef Mon Sep 17 00:00:00 2001 From: Yutaro Iiyama Date: Sat, 7 Oct 2017 11:15:14 -0400 Subject: [PATCH 1/5] Max's updates of the registry tables --- etc/db/dynamoregister.sql | 48 +++++++++++++++++++++++++++++++++++---- 1 file changed, 43 insertions(+), 5 deletions(-) diff --git a/etc/db/dynamoregister.sql b/etc/db/dynamoregister.sql index 9ec88a8a..643d34d3 100644 --- a/etc/db/dynamoregister.sql +++ b/etc/db/dynamoregister.sql @@ -2,7 +2,7 @@ -- -- Host: localhost Database: dynamoregister -- ------------------------------------------------------ --- Server version 5.1.73 +-- Server version 5.1.73-log /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; /*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; @@ -55,10 +55,11 @@ DROP TABLE IF EXISTS `deletion_queue`; /*!40101 SET @saved_cs_client = @@character_set_client */; /*!40101 SET character_set_client = utf8 */; CREATE TABLE `deletion_queue` ( + `reqid` int(10) unsigned NOT NULL DEFAULT '0', `file` varchar(512) COLLATE latin1_general_cs NOT NULL, - `target` varchar(64) COLLATE latin1_general_cs NOT NULL, - `created` datetime NOT NULL, - UNIQUE KEY `files` (`file`,`target`) + `site` varchar(32) COLLATE latin1_general_cs NOT NULL, + `status` enum('new','done','failed') COLLATE latin1_general_cs NOT NULL, + UNIQUE KEY `file` (`file`,`site`) ) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_general_cs; /*!40101 SET character_set_client = @saved_cs_client */; @@ -103,6 +104,43 @@ CREATE TABLE `domains` ( ) ENGINE=MyISAM AUTO_INCREMENT=1 DEFAULT CHARSET=latin1 COLLATE=latin1_general_cs; /*!40101 SET character_set_client = @saved_cs_client */; +-- +-- Table structure for table `requests` +-- + +DROP TABLE IF EXISTS `requests`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `requests` ( + `item` varchar(512) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + `datatype` enum('dataset','block') CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + `site` varchar(32) NOT NULL, + `reqtype` enum('copy','delete') NOT NULL, + `created` datetime NOT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `requests_unified` +-- + +DROP TABLE IF EXISTS `requests_unified`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `requests_unified` ( + `reqid` int(10) unsigned NOT NULL AUTO_INCREMENT, + `item` varchar(512) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + `datatype` enum('dataset','block') CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + `site` varchar(32) NOT NULL, + `reqtype` enum('copy','delete') NOT NULL, + `rank` int(10) unsigned DEFAULT '0', + `status` enum('new','queued') NOT NULL, + `created` datetime NOT NULL, + `updated` datetime DEFAULT NULL, + PRIMARY KEY (`reqid`) +) ENGINE=MyISAM AUTO_INCREMENT=1 DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + -- -- Table structure for table `services` -- @@ -162,4 +200,4 @@ CREATE TABLE `users` ( /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; --- Dump completed on 2017-06-26 15:52:45 +-- Dump completed on 2017-10-07 11:12:50 From 15a9003cf293466188ca1272a9677d346ec035a4 Mon Sep 17 00:00:00 2001 From: Yutaro Iiyama Date: Sat, 7 Oct 2017 11:18:28 -0400 Subject: [PATCH 2/5] adding the invalidations table --- etc/db/dynamoregister.sql | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/etc/db/dynamoregister.sql b/etc/db/dynamoregister.sql index 643d34d3..d9f59074 100644 --- a/etc/db/dynamoregister.sql +++ b/etc/db/dynamoregister.sql @@ -104,6 +104,20 @@ CREATE TABLE `domains` ( ) ENGINE=MyISAM AUTO_INCREMENT=1 DEFAULT CHARSET=latin1 COLLATE=latin1_general_cs; /*!40101 SET character_set_client = @saved_cs_client */; +-- +-- Table structure for table `invalidations` +-- + +DROP TABLE IF EXISTS `invalidations`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `invalidations` ( + `item` varchar(512) CHARACTER SET latin1 COLLATE latin1_general_cs NOT NULL, + `user_id` int(10) unsigned NOT NULL DEFAULT '0', + `timestamp` datetime NOT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + -- -- Table structure for table `requests` -- From fda5b6409123552bd02ded09522c75db711ed833 Mon Sep 17 00:00:00 2001 From: Yutaro Iiyama Date: Tue, 10 Oct 2017 07:58:48 -0400 Subject: [PATCH 3/5] putting /*/Express/* back into Express partition definition --- lib/common/configuration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/common/configuration.py b/lib/common/configuration.py index 45f35961..8958a82f 100644 --- a/lib/common/configuration.py +++ b/lib/common/configuration.py @@ -184,7 +184,7 @@ ('IB RelVal', lambda r: r.group is not None and r.group.name == 'IB RelVal'), ('Tape', lambda r: r.site.storage_type == Site.TYPE_MSS), ('Unsubscribed', lambda r: r.group is None), - ('Express', lambda r: r.group is not None and r.group.name == 'express' and re.match('/.*Express.*/.+/.+$', r.block.dataset.name)), + ('Express', lambda r: r.group is not None and r.group.name == 'express' and re.match('/(?:.*Express.*/.+|.+/.*Express.*)/.+$', r.block.dataset.name)), ('Physics', ['AnalysisOps', 'DataOps']), ], # list of conditions for a PRODUCTION state dataset to become IGNORED (will still be reset to PRODUCTION if a new block replica is found) From 329c164bbcc04aaaba0804dd340c904d26069e6f Mon Sep 17 00:00:00 2001 From: Yutaro Iiyama Date: Tue, 10 Oct 2017 08:02:50 -0400 Subject: [PATCH 4/5] policy update --- etc/policies.tag | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/policies.tag b/etc/policies.tag index 1cf14735..2e5673d9 100644 --- a/etc/policies.tag +++ b/etc/policies.tag @@ -1 +1 @@ -v49 +v50 From 43c4081c6bd0b51c35389902100d6d49ae5538dc Mon Sep 17 00:00:00 2001 From: Yutaro Iiyama Date: Tue, 10 Oct 2017 08:03:05 -0400 Subject: [PATCH 5/5] prototype file invalidation API --- web/cgi-bin/registry/invalidation.php | 62 +++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 web/cgi-bin/registry/invalidation.php diff --git a/web/cgi-bin/registry/invalidation.php b/web/cgi-bin/registry/invalidation.php new file mode 100644 index 00000000..90ee8252 --- /dev/null +++ b/web/cgi-bin/registry/invalidation.php @@ -0,0 +1,62 @@ + /command + +if ($command == "") { + # show webpage + exit(0); +} + +if ($command == 'help') { +# echo file_get_contents(__DIR__ . '/html/invalidate_help.html'); + exit(0); +} + +include_once(__DIR__ . '/../dynamo/common/db_conf.php'); +include_once(__DIR__ . '/common.php'); + +/* if ($_SERVER['SSL_CLIENT_VERIFY'] != 'SUCCESS') */ +/* send_response(401, 'AuthFailed', 'SSL authentication failed.'); */ + +if ($command == 'invalidate' || $command == 'clear') { + if (!isset($_REQUEST['item'])) + send_response(400, 'BadRequest', '\'item\' field is required', NULL, 'json'); + + $item = $_REQUEST['item']; + + $db = new mysqli($db_conf['host'], $db_conf['user'], $db_conf['password'], 'dynamoregister'); + + if ($command == 'invalidate') { + $stmt = $db->prepare('INSERT INTO `invalidations` (`item`, `timestamp`) VALUES (?, NOW())'); + $message = 'Item added'; + } + else { + $stmt = $db->prepare('DELETE FROM `invalidations` WHERE `item` = ?'); + $message = 'Item removed'; + } + + $return_data = array(); + + if (is_array($item)) { + foreach ($item as $t) { + $stmt->bind_param('s', $t); + $stmt->execute(); + $return_data[] = array('item' => $t); + } + } + else { + $stmt->bind_param('s', $item); + $stmt->execute(); + $return_data[] = array('item' => $item); + } + $stmt->close(); + + send_response(200, 'OK', $message, $return_data, 'json'); +} +else { + send_response(400, 'BadRequest', 'Invalid command (possible values: help, invalidate)'); +} + +?> \ No newline at end of file