Add support for transition period and check for incompatible OJ certificates

DEVSIX-9285

Autoported commit.
Original commit hash: [f33d3c977]

Author: Eugene Bochilo

itext.tests/itext.sign.tests/itext/signatures/validation/lotl/LotlServiceTest.cs

@@ -109,7 +109,7 @@ public virtual void TestCacheFailCallsDownloadOfPivotFile() {
                 lotlService.WithLotlServiceCache(new LotlServiceTest.CacheReturnsNull());
                 lotlService.WithPivotFetcher(new _PivotFetcher_125(lotlService));
                 f = lotlService.GetAndValidatePivotFiles("abc".GetBytes(System.Text.Encoding.UTF8), JavaCollectionsUtil.EmptyList
-                    <IX509Certificate>());
+                    <IX509Certificate>(), null);
             }
             NUnit.Framework.Assert.IsNotNull(f);
             NUnit.Framework.Assert.AreEqual(1, f.GetPivotUrls().Count);
@@ -202,7 +202,7 @@ public virtual void TestCacheRefreshIsFiring() {
             AtomicLong refreshCounter = new AtomicLong(0);
             using (LotlService lotlService = new _LotlService_207(refreshCounter, lotlFetchingProperties)) {
                 lotlService.SetupTimer();
-                Thread.Sleep(1000);
+                Thread.Sleep(2000);
             }
             NUnit.Framework.Assert.IsTrue(refreshCounter.Get() >= 8, "Refresh counter should be greater than 8, but was: "
                  + refreshCounter.Get());

itext.tests/itext.sign.tests/itext/signatures/validation/lotl/LotlValidatorTest.cs

@@ -192,6 +192,9 @@ public sealed class SignExceptionMessageConstant {
         public const String CACHE_ALREADY_INITIALIZED = "Global LOTL service has already been initialized. " + "You cannot initialize it again. If you want to use a different configuration, please create a new "
              + "instance of LotlService with the desired properties and use it in ValidatorChainBuilder.";
 
+        public const String OFFICIAL_JOURNAL_CERTIFICATES_OUTDATED = "Trusted certificates from Official Journal of European Union are outdated. "
+             + "LOTL file cannot be validated. " + "Please, provide OJ certificates, which match the ones used to sign European Union List of Trusted Lists.";
+
         private SignExceptionMessageConstant() {
         }
         // Private constructor will prevent the instantiation of this class directly

itext/itext.sign/itext/signatures/exceptions/SignExceptionMessageConstant.cs

@@ -50,6 +50,10 @@ public sealed class SignLogMessageConstant {
 
         public const String FAILED_TO_FETCH_EU_JOURNAL_CERTIFICATES = "Problem occurred while fetching " + "EU Journal certificates.\n{0}";
 
+        public const String OJ_TRANSITION_PERIOD = "Main LOTL file contains two Official Journal of European Union links. "
+             + "This usually indicates that transition period for Official Journal has started. " + "Newest version of Official Journal should be used from now on "
+             + "to retrieve trusted certificates and LOTL location.";
+
         private SignLogMessageConstant() {
         }
         // Private constructor will prevent the instantiation of this class directly

itext/itext.sign/itext/signatures/logs/SignLogMessageConstant.cs

@@ -20,6 +20,7 @@ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 You should have received a copy of the GNU Affero General Public License
 along with this program.  If not, see <https://www.gnu.org/licenses/>.
 */
+using System;
 using System.Collections.Generic;
 using iText.Commons.Bouncycastle.Cert;
 using iText.Signatures.Validation;
@@ -46,6 +47,7 @@ public EuropeanResourceFetcher() {
         public virtual EuropeanResourceFetcher.Result GetEUJournalCertificates() {
             EuropeanResourceFetcher.Result result = new EuropeanResourceFetcher.Result();
             EuropeanTrustedListConfigurationFactory factory = EuropeanTrustedListConfigurationFactory.GetFactory()();
+            result.SetCurrentlySupportedPublication(factory.GetCurrentlySupportedPublication());
             SafeCalling.OnExceptionLog(() => result.SetCertificates(factory.GetCertificates()), result.GetLocalReport(
                 ), (e) => new ReportItem(LotlValidator.LOTL_VALIDATION, LotlValidator.JOURNAL_CERT_NOT_PARSABLE, e, ReportItem.ReportItemStatus
                 .INFO));
@@ -62,6 +64,8 @@ public class Result {
 
             private IList<IX509Certificate> certificates;
 
+            private String currentlySupportedPublication;
+
             /// <summary>
             /// Create a new Instance of
             /// <see cref="Result"/>.
@@ -83,11 +87,31 @@ public virtual IList<IX509Certificate> GetCertificates() {
                 return certificates;
             }
 
+            /// <summary>Gets string constant representing currently used Official Journal publication.</summary>
+            /// <returns>
+            /// 
+            /// <see cref="System.String"/>
+            /// constant representing currently used Official Journal publication
+            /// </returns>
+            public virtual String GetCurrentlySupportedPublication() {
+                return currentlySupportedPublication;
+            }
+
             /// <summary>Sets the list of certificates.</summary>
             /// <param name="certificates">a list of Certificate objects to set</param>
             public virtual void SetCertificates(IList<IX509Certificate> certificates) {
                 this.certificates = certificates;
             }
+
+            /// <summary>Sets string constant representing currently used Official Journal publication.</summary>
+            /// <param name="currentlySuppostedPublication">
+            /// 
+            /// <see cref="System.String"/>
+            /// constant representing currently used Official Journal publication
+            /// </param>
+            public virtual void SetCurrentlySupportedPublication(String currentlySuppostedPublication) {
+                this.currentlySupportedPublication = currentlySuppostedPublication;
+            }
         }
     }
 }

itext/itext.sign/itext/signatures/validation/lotl/EuropeanResourceFetcher.cs

@@ -178,7 +178,7 @@ public virtual iText.Signatures.Validation.Lotl.LotlFetchingProperties SetCacheS
         /// Gets the calculation function for the cache refresh interval.
         /// <para />
         /// This function will be used to determine the refresh interval based on the staleness time.
-        /// By default, it takes 70% of the staleness time as the refresh interval.
+        /// By default, it takes 23% of the staleness time as the refresh interval.
         /// </remarks>
         /// <returns>
         /// a function that takes the staleness time in milliseconds and returns the refresh interval in

itext/itext.sign/itext/signatures/validation/lotl/LotlFetchingProperties.cs

@@ -178,12 +178,14 @@ public virtual void InitializeCache() {
             SetupTimer();
             EuropeanLotlFetcher.Result mainLotlResult = lotlByteFetcher.Fetch();
             if (!mainLotlResult.GetLocalReport().GetFailures().IsEmpty()) {
-                //We throw on main Lotl fetch failure, so we don't proceed to pivot and country specific LOTL fetches
+                // We throw on main LOTL fetch failure, so we don't proceed to pivot and country specific LOTL fetches
                 ReportItem reportItem = mainLotlResult.GetLocalReport().GetFailures()[0];
                 throw new PdfException(reportItem.GetMessage(), reportItem.GetExceptionCause());
             }
             EuropeanResourceFetcher.Result europeanResourceFetcherEUJournalCertificates = europeanResourceFetcher.GetEUJournalCertificates
                 ();
+            pivotFetcher.SetCurrentJournalUri(europeanResourceFetcherEUJournalCertificates.GetCurrentlySupportedPublication
+                ());
             PivotFetcher.Result pivotsResult = pivotFetcher.DownloadAndValidatePivotFiles(mainLotlResult.GetLotlXml(), 
                 europeanResourceFetcherEUJournalCertificates.GetCertificates());
             if (!pivotsResult.GetLocalReport().GetFailures().IsEmpty()) {
@@ -359,9 +361,11 @@ protected internal virtual void TryAndRefreshCache() {
             EuropeanLotlFetcher.Result mainLotlResult = null;
             bool mainLotlFetchSuccessful = false;
             Exception mainLotlFetchException = null;
+            String currentJournalUri;
             try {
                 EuropeanResourceFetcher.Result europeanResourceFetcherEUJournalCertificates = europeanResourceFetcher.GetEUJournalCertificates
                     ();
+                currentJournalUri = europeanResourceFetcherEUJournalCertificates.GetCurrentlySupportedPublication();
                 if (europeanResourceFetcherEUJournalCertificates.GetLocalReport().GetValidationResult() != ValidationReport.ValidationResult
                     .VALID) {
                     throw new PdfException(MessageFormatUtil.Format(SignExceptionMessageConstant.FAILED_TO_FETCH_EU_JOURNAL_CERTIFICATES
@@ -388,6 +392,7 @@ protected internal virtual void TryAndRefreshCache() {
             if (mainLotlFetchSuccessful) {
                 //Only if the main Lotl was fetched successfully, we proceed to re-fetch the new pivot files.
                 try {
+                    pivotFetcher.SetCurrentJournalUri(currentJournalUri);
                     pivotResult = pivotFetcher.DownloadAndValidatePivotFiles(mainLotlResult.GetLotlXml(), europeanResourceFetcher
                         .GetEUJournalCertificates().GetCertificates());
                     fetchPivotFilesSuccessful = pivotResult.GetLocalReport().GetValidationResult() == ValidationReport.ValidationResult
@@ -440,11 +445,12 @@ protected internal virtual void TryAndRefreshCache() {
 
 //\cond DO_NOT_DOCUMENT
         internal virtual PivotFetcher.Result GetAndValidatePivotFiles(byte[] lotlXml, IList<IX509Certificate> certificates
-            ) {
+            , String currentJournalUri) {
             PivotFetcher.Result result = cache.GetPivotResult();
             if (result != null) {
                 return result;
             }
+            pivotFetcher.SetCurrentJournalUri(currentJournalUri);
             PivotFetcher.Result newResult = pivotFetcher.DownloadAndValidatePivotFiles(lotlXml, certificates);
             cache.SetPivotResult(newResult);
             return newResult;

itext/itext.sign/itext/signatures/validation/lotl/LotlService.cs

@@ -86,7 +86,7 @@ public virtual ValidationReport Validate() {
             EuropeanResourceFetcher.Result europeanResult = service.GetEUJournalCertificates();
             report.Merge(europeanResult.GetLocalReport());
             PivotFetcher.Result result = service.GetAndValidatePivotFiles(lotl.GetLotlXml(), europeanResult.GetCertificates
-                ());
+                (), europeanResult.GetCurrentlySupportedPublication());
             report.Merge(result.GetLocalReport());
             if (result.GetLocalReport().GetValidationResult() != ValidationReport.ValidationResult.VALID) {
                 return report;

itext/itext.sign/itext/signatures/validation/lotl/LotlValidator.cs

@@ -23,24 +23,48 @@ You should have received a copy of the GNU Affero General Public License
 using System;
 using System.Collections.Generic;
 using System.IO;
+using System.Linq;
+using Microsoft.Extensions.Logging;
+using iText.Commons;
 using iText.Commons.Bouncycastle.Cert;
 using iText.Commons.Utils;
 using iText.Kernel.Exceptions;
+using iText.Signatures.Exceptions;
+using iText.Signatures.Logs;
 using iText.Signatures.Validation;
 using iText.Signatures.Validation.Lotl.Xml;
 using iText.Signatures.Validation.Report;
 
 namespace iText.Signatures.Validation.Lotl {
     /// <summary>This class fetches and validates pivot files from a List of Trusted Lists (Lotl) XML.</summary>
     public class PivotFetcher {
+        private static readonly ILogger LOGGER = ITextLogManager.GetLogger(typeof(iText.Signatures.Validation.Lotl.PivotFetcher
+            ));
+
         private readonly LotlService service;
 
+        private String currentJournalUri;
+
         /// <summary>Constructs a PivotFetcher with the specified LotlService and ValidatorChainBuilder.</summary>
         /// <param name="service">the LotlService used to retrieve resources</param>
         public PivotFetcher(LotlService service) {
             this.service = service;
         }
 
+        /// <summary>
+        /// Sets
+        /// <see cref="System.String"/>
+        /// constant representing currently used Official Journal publication.
+        /// </summary>
+        /// <param name="currentJournalUri">
+        /// 
+        /// <see cref="System.String"/>
+        /// constant representing currently used Official Journal publication
+        /// </param>
+        public virtual void SetCurrentJournalUri(String currentJournalUri) {
+            this.currentJournalUri = currentJournalUri;
+        }
+
         /// <summary>Fetches and validates pivot files from the provided Lotl XML.</summary>
         /// <param name="lotlXml">the byte array of the Lotl XML</param>
         /// <param name="certificates">the list of trusted certificates</param>
@@ -50,20 +74,31 @@ public virtual PivotFetcher.Result DownloadAndValidatePivotFiles(byte[] lotlXml,
             if (lotlXml == null) {
                 throw new PdfException(LotlValidator.UNABLE_TO_RETRIEVE_LOTL);
             }
-            XmlPivotsHandler pivotsHandler = new XmlPivotsHandler();
-            new XmlSaxProcessor().Process(new MemoryStream(lotlXml), pivotsHandler);
             PivotFetcher.Result result = new PivotFetcher.Result();
-            IList<String> pivotsUrlList = pivotsHandler.GetPivots();
+            IList<String> pivotsUrlList = GetPivotsUrlList(lotlXml);
+            IList<String> ojUris = pivotsUrlList.Where((url) => XmlPivotsHandler.IsOfficialJournal(url)).ToList();
+            if (ojUris.Count > 1) {
+                LOGGER.LogWarning(SignLogMessageConstant.OJ_TRANSITION_PERIOD);
+            }
             result.SetPivotUrls(pivotsUrlList);
             IList<byte[]> pivotFiles = new List<byte[]>();
+            // If we weren't able to find any OJ links, or current OJ uri is null, we process all the pivots.
+            bool startProcessing = ojUris.IsEmpty() || currentJournalUri == null;
             // We need to process pivots backwards.
             for (int i = pivotsUrlList.Count - 1; i >= 0; i--) {
                 String pivotUrl = pivotsUrlList[i];
-                SafeCalling.OnExceptionLog(() => pivotFiles.Add(service.GetResourceRetriever().GetByteArrayByUrl(new Uri(pivotUrl
-                    ))), result.GetLocalReport(), (e) => new ReportItem(LotlValidator.LOTL_VALIDATION, MessageFormatUtil.Format
-                    (LotlValidator.UNABLE_TO_RETRIEVE_PIVOT, pivotUrl), e, ReportItem.ReportItemStatus.INVALID));
-                if (result.GetLocalReport().GetValidationResult() != ValidationReport.ValidationResult.VALID) {
-                    return result;
+                if (pivotUrl.Equals(currentJournalUri)) {
+                    // We only need to process pivots which, were created after OJ entry was added.
+                    startProcessing = true;
+                    continue;
+                }
+                if (startProcessing && !XmlPivotsHandler.IsOfficialJournal(pivotUrl)) {
+                    SafeCalling.OnExceptionLog(() => pivotFiles.Add(service.GetResourceRetriever().GetByteArrayByUrl(new Uri(pivotUrl
+                        ))), result.GetLocalReport(), (e) => new ReportItem(LotlValidator.LOTL_VALIDATION, MessageFormatUtil.Format
+                        (LotlValidator.UNABLE_TO_RETRIEVE_PIVOT, pivotUrl), e, ReportItem.ReportItemStatus.INVALID));
+                    if (result.GetLocalReport().GetValidationResult() != ValidationReport.ValidationResult.VALID) {
+                        return result;
+                    }
                 }
             }
             IList<IX509Certificate> trustedCertificates = certificates;
@@ -82,6 +117,9 @@ public virtual PivotFetcher.Result DownloadAndValidatePivotFiles(byte[] lotlXml,
                     result.GetLocalReport().AddReportItem(new ReportItem(LotlValidator.LOTL_VALIDATION, LotlValidator.LOTL_VALIDATION_UNSUCCESSFUL
                         , ReportItem.ReportItemStatus.INVALID));
                     result.GetLocalReport().Merge(localReport);
+                    if (!ojUris.Any((ojUri) => ojUri.Equals(currentJournalUri))) {
+                        throw new PdfException(SignExceptionMessageConstant.OFFICIAL_JOURNAL_CERTIFICATES_OUTDATED);
+                    }
                     return result;
                 }
                 XmlCertificateRetriever certificateRetriever = new XmlCertificateRetriever(new XmlDefaultCertificateHandler
@@ -91,6 +129,19 @@ public virtual PivotFetcher.Result DownloadAndValidatePivotFiles(byte[] lotlXml,
             return result;
         }
 
+        /// <summary>Gets list of pivots xml files, including OJ entries.</summary>
+        /// <param name="lotlXml">
+        /// 
+        /// <c>byte</c>
+        /// array representing main LOTL file
+        /// </param>
+        /// <returns>list of pivots xml files, including OJ entries</returns>
+        protected internal virtual IList<String> GetPivotsUrlList(byte[] lotlXml) {
+            XmlPivotsHandler pivotsHandler = new XmlPivotsHandler();
+            new XmlSaxProcessor().Process(new MemoryStream(lotlXml), pivotsHandler);
+            return pivotsHandler.GetPivots();
+        }
+
         /// <summary>Result class encapsulates the result of the pivot fetching and validation process.</summary>
         public class Result {
             private ValidationReport localReport = new ValidationReport();

itext/itext.sign/itext/signatures/validation/lotl/PivotFetcher.cs

@@ -55,8 +55,11 @@ public virtual void EndElement(String uri, String localName, String qName) {
                 schemeInformationContext = false;
             }
             else {
-                if (XmlTagConstants.URI.Equals(localName) && IsPivot(uriLink.ToString())) {
-                    pivots.Add(uriLink.ToString());
+                if (XmlTagConstants.URI.Equals(localName)) {
+                    String uriLinkString = uriLink.ToString();
+                    if (IsPivot(uriLinkString) || IsOfficialJournal(uriLinkString)) {
+                        pivots.Add(uriLinkString);
+                    }
                 }
             }
         }
@@ -71,6 +74,12 @@ public virtual IList<String> GetPivots() {
             return new List<String>(pivots);
         }
 
+//\cond DO_NOT_DOCUMENT
+        internal static bool IsOfficialJournal(String uriLink) {
+            return uriLink.Contains("eur-lex.europa.eu");
+        }
+//\endcond
+
         private static bool IsPivot(String uriLink) {
             return uriLink.Contains("eu-lotl-pivot");
         }