Permalink
Browse files

Merge branch 'staging'

Conflicts:
	src/NuGetGallery.Backend/WorkerRole.cs
  • Loading branch information...
2 parents b394324 + 13283b9 commit b612bcf90f60e6ffb792e2265f683a6c07a81f5d @anurse anurse committed Jan 29, 2014
Showing with 1,597 additions and 0 deletions.
  1. +143 −0 src/NuGetGallery.Backend/Dashboard.html
  2. BIN src/NuGetGallery.Backend/GlobalSuppressions.cs
  3. +160 −0 src/NuGetGallery.Backend/JobRunner.cs
  4. +29 −0 src/NuGetGallery.Backend/Jobs/BackupDatabaseJob.cs
  5. +40 −0 src/NuGetGallery.Backend/Jobs/BackupPackagesJob.cs
  6. +59 −0 src/NuGetGallery.Backend/Jobs/BackupWarehouseJob.cs
  7. +30 −0 src/NuGetGallery.Backend/Jobs/CleanJobLogsJob.cs
  8. +39 −0 src/NuGetGallery.Backend/Jobs/CleanOfflineDatabaseBackupsJob.cs
  9. +41 −0 src/NuGetGallery.Backend/Jobs/CleanOnlineDatabaseBackupsJob.cs
  10. +31 −0 src/NuGetGallery.Backend/Jobs/CreateWarehouseReportsJob.cs
  11. +43 −0 src/NuGetGallery.Backend/Jobs/ExecuteAggregateStatisticsJob.cs
  12. +42 −0 src/NuGetGallery.Backend/Jobs/ExportDailyBackupsJob.cs
  13. +42 −0 src/NuGetGallery.Backend/Jobs/HandleFailedPackageEditsJob.cs
  14. +29 −0 src/NuGetGallery.Backend/Jobs/HandleQueuedPackageEditsJob.cs
  15. +39 −0 src/NuGetGallery.Backend/Jobs/PurgePackageStatisticsJob.cs
  16. +44 −0 src/NuGetGallery.Backend/Jobs/ReplicatePackageStatisticsJob.cs
  17. +34 −0 src/NuGetGallery.Backend/Jobs/UpdateLicenseReportsJob.cs
  18. +85 −0 src/NuGetGallery.Backend/Jobs/WorkerJob.cs
  19. +46 −0 src/NuGetGallery.Backend/JsonLayout.cs
  20. +149 −0 src/NuGetGallery.Backend/NuGetGallery.Backend.csproj
  21. +5 −0 src/NuGetGallery.Backend/Properties/AssemblyInfo.cs
  22. +146 −0 src/NuGetGallery.Backend/Settings.cs
  23. +301 −0 src/NuGetGallery.Backend/WorkerRole.cs
  24. +3 −0 src/NuGetGallery.Backend/app.config
  25. +17 −0 src/NuGetGallery.Backend/packages.config
@@ -0,0 +1,143 @@
+<!DOCTYPE html>
+<html>
+<head>
+ <style>
+ body {
+ background-color:azure;
+ }
+ #header {
+ font-family:"Trebuchet MS", Arial, Helvetica, sans-serif;
+ text-align:center;
+ color:#000000;
+ }
+ #jobs {
+ font-family:"Trebuchet MS", Arial, Helvetica, sans-serif;
+ width:100%;
+ border-collapse:collapse;
+ }
+ #jobs td, #jobs th
+ {
+ font-size:1em;
+ border:1px solid #98bf21;
+ padding:3px 7px 2px 7px;
+ }
+ #jobs th {
+ font-size:1.2em;
+ text-align:left;
+ padding-top:5px;
+ padding-bottom:4px;
+ background-color:#A7C942;
+ color:#ffffff;
+ }
+ #jobs tr.alt {
+ color:#000000;
+ background-color:#EAF2D3;
+ }
+ #jobs td {
+ vertical-align:top;
+ }
+ .jobname {
+ width:150px;
+ font-weight:bold;
+ }
+ .statussuccess {
+ color:#ffffff;
+ background-color:green;
+ text-align:center;
+ }
+ .statusfailure {
+ color:#ffffff;
+ background-color:red;
+ text-align:center;
+ }
+ </style>
+
+ <title>NuGet Operations Dashboard</title>
+ <script src="http://ajax.microsoft.com/ajax/jquery/jquery-1.8.3.min.js" type="text/javascript"></script>
+
+ <script type="text/JavaScript">
+
+ var makeJobTableHtml = function (data) {
+
+ //alert(JSON.stringify(data));
+
+ var html = '';
+
+ html += '<table id="jobs">';
+
+ html += '<th style="width:150px">&nbsp;Job</th>';
+ html += '<th style="width:50px">PID</th>';
+ html += '<th style="width:150px">&nbsp;Time (UTC)</th>';
+ html += '<th style="width:80px">Duration</th>';
+ html += '<th style="width:60px">Status</th>';
+ html += '<th>&nbsp;Message</th>';
+
+ var rowClass = 'alt';
+
+ for (var job in data) {
+
+ var jobEntry = data[job];
+
+ if (rowClass == 'alt') {
+ rowClass = '';
+ }
+ else {
+ rowClass = 'alt';
+ }
+
+ for (var i = 0; i < jobEntry.length; i += 1) {
+
+ html += '<tr class="' + rowClass + '">';
+
+ if (i == 0) {
+ html += '<td class="jobname" rowspan="' + jobEntry.length + '">';
+ html += job;
+ html += '</td>';
+ }
+
+ html += '<td>' + jobEntry[i].pid + '</td>';
+
+ var at = jobEntry[i].at.replace(/ /g, '&nbsp;');
+ html += '<td>' + at + '</td>';
+
+ html += '<td>' + jobEntry[i].duration + '</td>';
+
+ var statusclass = 'statusfailure';
+ if (jobEntry[i].status == 'success') {
+ statusclass = 'statussuccess';
+ }
+
+ html += '<td class="' + statusclass + '">' + jobEntry[i].status + '</td>';
+
+ html += '<td style="">' + jobEntry[i].message + '</td>';
+
+ html += '</tr>';
+ }
+ }
+ html += '</table>';
+
+ return html;
+ }
+
+ var fetch = function () {
+ var address = 'http://' + $(location).attr('host') + '/ops/jobs.json';
+ $.getJSON(address, function (data, err) {
+ $('#results').html(makeJobTableHtml(data));
+ });
+ }
+
+ $(document).ready(function () {
+
+ fetch();
+ });
+
+ </script>
+</head>
+<body>
+
+ <h1 id="header">NuGet Operations Dashboard</h1>
+
+ <div id="results"></div>
+
+</body>
+</html>
Binary file not shown.
@@ -0,0 +1,160 @@
+using System;
+using System.Collections.Generic;
+using System.ComponentModel.Composition;
+using System.Linq;
+using System.Reactive;
+using System.Reactive.Linq;
+using System.Reactive.Subjects;
+using NLog;
+using NuGetGallery.Backend.Jobs;
+
+namespace NuGetGallery.Backend
+{
+ [Export]
+ public class JobRunner : IDisposable
+ {
+ private AsyncSubject<Unit> _subject = new AsyncSubject<Unit>();
+ private Logger _logger = LogManager.GetLogger("JobRunner");
+ private Settings _settings;
+
+ public IDictionary<string, WorkerJob> Jobs { get; private set; }
+
+ [ImportingConstructor]
+ public JobRunner([Import(AllowDefault = true)] Settings settings, [ImportMany] IEnumerable<WorkerJob> jobs)
+ {
+ _settings = settings ?? new Settings();
+ Jobs = jobs.ToDictionary(j => j.GetType().Name, StringComparer.OrdinalIgnoreCase);
+ _subject.OnNext(Unit.Default);
+
+ foreach (var job in Jobs.Values)
+ {
+ try
+ {
+ job.Initialize(_settings);
+ }
+ catch (Exception ex)
+ {
+ _logger.ErrorException(String.Format("{2} Initializing '{0}': {1}", job.GetType().Name, ex.Message, ex.GetType().Name), ex);
+ }
+ }
+ }
+
+ public void Dispose()
+ {
+ _subject.Dispose();
+ GC.SuppressFinalize(this);
+ }
+
+ public void Stop()
+ {
+ _subject.OnCompleted();
+ _logger.Info("Stopped Job Runner");
+ }
+
+ public void Run()
+ {
+ Run(Jobs);
+ }
+
+ void Run(IDictionary<string, WorkerJob> jobs)
+ {
+ _logger.Info("Scheduling Jobs...");
+
+ // Set up the schedules
+ IDisposable[] tokens;
+ try
+ {
+ tokens = jobs.Select(job =>
+ {
+ var startTime = DateTimeOffset.UtcNow + job.Value.Offset;
+ _logger.Debug("Scheduling '{0}' to run every '{1}' starting at '{2}'.", job.Value.GetType().Name, job.Value.Period, startTime.ToLocalTime().ToString("yyyy-MM-dd HH:mm:ss"));
+ return Observable.Timer(startTime, job.Value.Period)
+ .Subscribe(_ => RunJob(job.Key, job.Value));
+ }).ToArray();
+ }
+ catch (Exception ex)
+ {
+ _logger.ErrorException(String.Format("Error scheduling jobs: {0}", ex.Message), ex);
+ return;
+ }
+
+ // Wait for a completion message
+ _logger.Info("Ready at {0}. Waiting for Shutdown", DateTime.Now);
+
+ // Wait for the system to shut down, but perform a heartbeat every minute.
+ using (Observable.Interval(TimeSpan.FromMinutes(1)).Subscribe(t => _logger.Info("Heartbeat tick. Host is still running.")))
+ {
+ _subject.Wait();
+ }
+
+ _logger.Info("Shutting down jobs...");
+ foreach (var token in tokens)
+ {
+ token.Dispose();
+ }
+ }
+
+ public void RunSingleJob(string name)
+ {
+ _logger.Info("Running " + name);
+
+ WorkerJob job;
+ if (!Jobs.TryGetValue(name, out job))
+ {
+ _logger.Error("No such job: " + name);
+ throw new InvalidOperationException("No such job: " + name);
+ }
+ RunJob(name, job);
+ }
+
+ public void RunSingleJobContinuously(string name)
+ {
+ WorkerJob job;
+ if (!Jobs.TryGetValue(name, out job))
+ {
+ _logger.Error("No such job: " + name);
+ throw new InvalidOperationException("No such job: " + name);
+ }
+
+ Run(new Dictionary<string, WorkerJob> { { name, job } });
+ }
+
+ public void OnStop()
+ {
+ foreach (var job in Jobs)
+ {
+ _logger.Debug("Cleaning up Job '{0}'", job.Key);
+ }
+ }
+
+ public bool OnStart()
+ {
+ foreach (var job in Jobs)
+ {
+ _logger.Debug("Initializing Job '{0}'", job.Key);
+ }
+ return true;
+ }
+
+ private void RunJob(string name, WorkerJob job)
+ {
+ try
+ {
+ _logger.Debug("Executing Job '{0}'", name);
+
+ try
+ {
+ job.RunOnce();
+ }
+ catch (Exception ex)
+ {
+ _logger.ErrorException(String.Format("Error Executing Job '{0}': {1}", name, ex.Message), ex);
+ }
+ }
+ catch (Exception ex)
+ {
+ _logger.ErrorException(String.Format("Infrastructure Error Executing Job '{0}': {1}", name, ex.Message), ex);
+ }
+ }
+ }
+}
@@ -0,0 +1,29 @@
+using System;
+using System.ComponentModel.Composition;
+using System.Data.SqlClient;
+using NuGetGallery.Operations;
+
+namespace NuGetGallery.Backend.Jobs
+{
+ [Export(typeof(WorkerJob))]
+ public class BackupDatabaseJob : WorkerJob
+ {
+ public override TimeSpan Period
+ {
+ get
+ {
+ return TimeSpan.FromMinutes(5);
+ }
+ }
+
+ public override void RunOnce()
+ {
+ ExecuteTask(new BackupDatabaseTask
+ {
+ ConnectionString = new SqlConnectionStringBuilder(Settings.MainConnectionString),
+ WhatIf = Settings.WhatIf,
+ IfOlderThan = 30,
+ });
+ }
+ }
+}
@@ -0,0 +1,40 @@
+using System;
+using System.ComponentModel.Composition;
+using System.Data.SqlClient;
+using NuGetGallery.Operations;
+
+namespace NuGetGallery.Backend.Jobs
+{
+ [Export(typeof(WorkerJob))]
+ public class BackupPackagesJob : WorkerJob
+ {
+ public override TimeSpan Period
+ {
+ get
+ {
+ return TimeSpan.FromMinutes(10);
+ }
+ }
+
+ public override TimeSpan Offset
+ {
+ get
+ {
+ return TimeSpan.FromMinutes(1);
+ }
+ }
+
+ public override void RunOnce()
+ {
+ Logger.Info("Starting backup packages task.");
+ ExecuteTask(new BackupPackagesTask
+ {
+ ConnectionString = new SqlConnectionStringBuilder(Settings.MainConnectionString),
+ StorageAccount = Settings.MainStorage,
+ BackupStorage = Settings.BackupStorage,
+ WhatIf = Settings.WhatIf
+ });
+ Logger.Info("Finished backup packages task.");
+ }
+ }
+}
Oops, something went wrong.

0 comments on commit b612bcf

Please sign in to comment.