Implement geoIP feature for zypp

This patch adds a feature to rewrite request URLs to the repo servers by querying a geoIP file from download.opensuse.org. This file can return a redirection target depending on the clients IP adress, this way we can directly contact a local mirror of d.o.o instead. The redir target stays valid for 24hrs.
openSUSE · Oct 13, 2022 · 91e3dd2 · 91e3dd2
1 parent 3f9c20a
commit 91e3dd2
Show file tree

Hide file tree

Showing 11 changed files with 318 additions and 25 deletions.
diff --git a/zypp.conf b/zypp.conf
@@ -257,6 +257,21 @@
 ##
 ## download.media_mountdir = /var/adm/mount
 
+##
+## Whether to use the geoip feature of download.opensuse.org
+##
+## Valid values: boolean
+## Default value: true
+##
+## The media backend can rewrite download requests to the geographically closest availble mirror.
+## Which exact mirror is used will be determined by requesting a "geoip" file from download.opensuse.org
+## via a HTTP GET request to https://download.opensuse.org/geoip , the server will use the clients IP to
+## determine the closest mirror if available.
+## Some specific files are however excluded from this redirection due to security reasons, especially the
+## repo metdata index and it's key and checksum files: repomd.xml, repomd.xml.key and repomd.xml.asc
+##
+## download.use_geoip_mirror = true
+
 ##
 ## Signature checking (repo metadata and downloaded rpm packages)
 ##

diff --git a/zypp/CMakeLists.txt b/zypp/CMakeLists.txt
@@ -248,7 +248,7 @@ SET( zypp_media_SRCS
   media/MediaDIR.cc
   media/MediaDISK.cc
   media/MediaCIFS.cc
-  media/MediaNetworkCommonHandler.h
+  media/MediaNetworkCommonHandler.cc
   media/MediaCurl.cc
   media/MediaMultiCurl.cc
   media/MediaISO.cc
@@ -265,6 +265,7 @@ SET( zypp_media_HEADERS
   media/MediaCIFS.h
   media/MediaCurl.h
   media/MediaMultiCurl.h
+  media/MediaNetworkCommonHandler.h
   media/MediaDIR.h
   media/MediaDISK.h
   media/MediaException.h

diff --git a/zypp/RepoManager.cc b/zypp/RepoManager.cc
@@ -17,6 +17,7 @@
 #include <list>
 #include <map>
 #include <algorithm>
+#include <chrono>
 
 #include <solv/solvversion.h>
 
@@ -42,6 +43,7 @@
 
 #include <zypp/parser/RepoFileReader.h>
 #include <zypp/parser/ServiceFileReader.h>
+#include <zypp/parser/xml/Reader.h>
 #include <zypp/repo/ServiceRepos.h>
 #include <zypp/repo/yum/Downloader.h>
 #include <zypp/repo/susetags/Downloader.h>
@@ -668,6 +670,8 @@ namespace zypp
 
     repo::ServiceType probeService( const Url & url ) const;
 
+    void refreshGeoIPData ();
+
   private:
     void saveService( ServiceInfo & service ) const;
 
@@ -999,6 +1003,8 @@ namespace zypp
     {
       MIL << "Check if to refresh repo " << info.alias() << " at " << url << " (" << info.type() << ")" << endl;
 
+      refreshGeoIPData();
+
       // first check old (cached) metadata
       Pathname mediarootpath = rawcache_path_for_repoinfo( _options, info );
       filesystem::assert_dir( mediarootpath );
@@ -1122,6 +1128,9 @@ namespace zypp
     assert_alias(info);
     assert_urls(info);
 
+    // make sure geoIP data is up 2 date
+    refreshGeoIPData();
+
     // we will throw this later if no URL checks out fine
     RepoException rexception( info, PL_("Valid metadata not found at specified URL",
                                         "Valid metadata not found at specified URLs",
@@ -1267,8 +1276,8 @@ namespace zypp
   {
     ProgressData progress(100);
     progress.sendTo(progressfnc);
-
-    filesystem::recursive_rmdir(rawcache_path_for_repoinfo(_options, info));
+    filesystem::recursive_rmdir( ZConfig::instance().geoipCachePath() );
+    filesystem::recursive_rmdir( rawcache_path_for_repoinfo(_options, info) );
     progress.toMax();
   }
 
@@ -2540,6 +2549,134 @@ namespace zypp
     return repo::ServiceType::NONE;
   }
 
+  void RepoManager::Impl::refreshGeoIPData ()
+  {
+    try  {
+
+      if ( !ZConfig::instance().geoipEnabled() ) {
+        MIL << "GeoIp disabled via ZConfig, not refreshing the GeoIP information." << std::endl;
+        return;
+      }
+
+      // for applications like packageKit that are running very long we remember the last time when we checked
+      // for geoIP data. We don't want to query this over and over again.
+      static auto lastCheck = std::chrono::steady_clock::time_point::min();
+      if ( lastCheck != std::chrono::steady_clock::time_point::min()
+          && (std::chrono::steady_clock::now() - lastCheck) < std::chrono::hours(24) )
+        return;
+
+      lastCheck = std::chrono::steady_clock::now();
+
+      const auto &geoIPCache = ZConfig::instance().geoipCachePath();
+
+      if ( filesystem::assert_dir( geoIPCache ) != 0 ) {
+        MIL << "Unable to create cache directory for GeoIP." << std::endl;
+        return;
+      }
+
+      if ( !PathInfo(geoIPCache).userMayRWX() ) {
+        MIL << "No access rights for the GeoIP cache directory." << std::endl;
+        return;
+      }
+
+      // remove all older cache entries
+      filesystem::dirForEachExt( geoIPCache, []( const Pathname &dir, const filesystem::DirEntry &entry ){
+        if ( entry.type != filesystem::FT_FILE )
+          return true;
+
+        PathInfo pi( dir/entry.name );
+        auto age = std::chrono::system_clock::now() - std::chrono::system_clock::from_time_t( pi.mtime() );
+        if ( age < std::chrono::hours(24) )
+          return true;
+
+        MIL << "Removing GeoIP file for " << entry.name << " since it's older than 24hrs." << std::endl;
+        filesystem::unlink( dir/entry.name );
+        return true;
+      });
+
+      // go over all configured hostnames
+      const auto &hosts = ZConfig::instance().geoipHostnames();
+      std::for_each( hosts.begin(), hosts.end(), [ & ]( const std::string &hostname ) {
+
+        // do not query files that are still there
+        if ( zypp::PathInfo( geoIPCache / hostname ).isExist() )  {
+          MIL << "Skipping GeoIP request for " << hostname << " since a valid cache entry exists." << std::endl;
+          return;
+        }
+
+        MIL << "Query GeoIP for " << hostname << std::endl;
+
+        zypp::Url url;
+        try
+        {
+          url.setHost(hostname);
+          url.setScheme("https");
+        }
+        catch(const zypp::Exception &e )
+        {
+          ZYPP_CAUGHT(e);
+          MIL << "Ignoring invalid GeoIP hostname: " << hostname << std::endl;
+          return;
+        }
+
+        zypp::ManagedFile file;
+        try {
+
+          // query the file from the server
+          MediaSetAccess acc( url );
+          file = zypp::ManagedFile (acc.provideOptionalFile("/geoip"), filesystem::unlink );
+
+        } catch ( const zypp::Exception &e  ) {
+          ZYPP_CAUGHT(e);
+          MIL << "Failed to query GeoIP from hostname: " << hostname << std::endl;
+          return;
+        }
+        if ( !file->empty() ) {
+
+          constexpr auto writeHostToFile = []( const Pathname &fName, const std::string &host ){
+            std::ofstream out;
+            out.open( fName.asString(), std::ios_base::trunc );
+            if ( out.is_open() ) {
+              out << host << std::endl;
+            } else {
+              MIL << "Failed to create/open GeoIP cache file " << fName << std::endl;
+            }
+          };
+
+          std::string geoipMirror;
+          try {
+            xml::Reader reader( *file );
+            if ( reader.seekToNode( 1, "host" ) ) {
+              const auto &str = reader.nodeText().asString();
+
+              // make a dummy URL to ensure the hostname is valid
+              zypp::Url testUrl;
+              testUrl.setHost(str);
+              testUrl.setScheme("https");
+
+              if ( testUrl.isValid() ) {
+                MIL << "Storing geoIP redirection: " << hostname << " -> " << str << std::endl;
+                geoipMirror = str;
+              }
+
+            } else {
+              MIL << "No host entry or empty file returned for GeoIP, remembering for 24hrs" << std::endl;
+            }
+          } catch ( const zypp::Exception &e ) {
+            ZYPP_CAUGHT(e);
+            MIL << "Empty or invalid GeoIP file, not requesting again for 24hrs" << std::endl;
+          }
+
+          writeHostToFile( geoIPCache / hostname, geoipMirror );
+        }
+      });
+
+    } catch ( const zypp::Exception &e ) {
+      ZYPP_CAUGHT(e);
+      MIL << "Failed to query GeoIP data." << std::endl;
+    }
+  }
+
   ///////////////////////////////////////////////////////////////////
   //
   //	CLASS NAME : RepoManager
@@ -2699,6 +2836,9 @@ namespace zypp
   void RepoManager::modifyService( const std::string & oldAlias, const ServiceInfo & service )
   { return _pimpl->modifyService( oldAlias, service ); }
 
+  void RepoManager::refreshGeoIp ()
+  { return _pimpl->refreshGeoIPData(); }
+
   ////////////////////////////////////////////////////////////////////////////
 
   std::ostream & operator<<( std::ostream & str, const RepoManager & obj )

diff --git a/zypp/RepoManager.h b/zypp/RepoManager.h
@@ -634,6 +634,12 @@ namespace zypp
     void modifyService( const ServiceInfo & service )
     { modifyService( service.alias(), service ); }
 
+    /*!
+     * Checks with configured geoIP servers ( usually download.opensuse.org ) if there is new geoip data available, caches the results
+     * in the metadata cache for 24hrs.
+     */
+    void refreshGeoIp ();
+
   private:
     /**
      * Functor thats filter RepoInfo by service which it belongs to.

diff --git a/zypp/ZConfig.cc b/zypp/ZConfig.cc
@@ -415,6 +415,8 @@ namespace zypp
         , pkgGpgCheck			( indeterminate )
         , apply_locks_file		( true )
         , pluginsPath			( "/usr/lib/zypp/plugins" )
+        , geoipEnabled ( true )
+        , geoipHosts { "download.opensuse.org" }
       {
         MIL << "libzypp: " LIBZYPP_VERSION_STRING << endl;
         if ( PathInfo(_parsedZyppConf).isExist() )
@@ -518,6 +520,9 @@ namespace zypp
                 {
                   download_mediaMountdir.restoreToDefault( Pathname(value) );
                 }
+                else if ( entry == "download.use_geoip_mirror") {
+                  geoipEnabled = str::strToBool( value, geoipEnabled );
+                }
                 else if ( entry == "commit.downloadMode" )
                 {
                   commit_downloadMode.set( deserializeDownloadMode( value ) );
@@ -712,6 +717,10 @@ namespace zypp
 
     Option<Pathname> pluginsPath;
 
+    bool geoipEnabled;
+
+    std::vector<std::string> geoipHosts;
+
     /* Other config singleton instances */
     MediaConfig &_mediaConf = MediaConfig::instance();
 
@@ -1024,6 +1033,18 @@ namespace zypp
   Pathname ZConfig::needrebootPath() const
   { return configPath()/"needreboot.d"; }
 
+  void ZConfig::setGeoipEnabled( bool enable )
+  { _pimpl->geoipEnabled = enable; }
+
+  bool ZConfig::geoipEnabled () const
+  { return _pimpl->geoipEnabled; }
+
+  Pathname ZConfig::geoipCachePath() const
+  { return builtinRepoCachePath()/"geoip.d"; }
+
+  const std::vector<std::string> ZConfig::geoipHostnames () const
+  { return _pimpl->geoipHosts; }
+
   Pathname ZConfig::varsPath() const
   {
     return ( _pimpl->cfg_vars_path.empty()

diff --git a/zypp/ZConfig.h b/zypp/ZConfig.h
@@ -223,6 +223,28 @@ namespace zypp
        */
       Pathname needrebootPath() const;
 
+      /**
+       * Enables or disables the use of the geoip feature of download.opensuse.org
+       */
+      void setGeoipEnabled( bool enable = true );
+
+      /**
+       * Returns true if zypp should use the geoip feature of download.opensuse.org
+       */
+      bool geoipEnabled () const;
+
+      /**
+       * Path where the geoip caches are kept (/var/cache/zypp/geoip)
+       */
+      Pathname geoipCachePath() const;
+
+      /**
+       * All hostnames we want to rewrite using the geoip feature. The \ref RepoManager
+       * will try to query each hostname via: https://hostname/geoip to receive a redirection
+       * target for requests to the given hostname. The geoip targets are cached in \ref geoipCachePath.
+       */
+      const std::vector<std::string> geoipHostnames () const;
+
       /**
        * Path containing custom repo variable definitions (configPath()/vars.d).
        * \see \ref zypp::repo::RepoVarExpand Repo variable expander

diff --git a/zypp/media/MediaCurl.cc b/zypp/media/MediaCurl.cc
@@ -626,20 +626,6 @@ void MediaCurl::releaseFrom( const std::string & ejectDev )
   disconnect();
 }
 
-Url MediaCurl::getFileUrl( const Pathname & filename_r ) const
-{
-  // Simply extend the URLs pathname. An 'absolute' URL path
-  // is achieved by encoding the leading '/' in an URL path:
-  //   URL: ftp://user@server		-> ~user
-  //   URL: ftp://user@server/		-> ~user
-  //   URL: ftp://user@server//		-> ~user
-  //   URL: ftp://user@server/%2F	-> /
-  //                         ^- this '/' is just a separator
-  Url newurl( _url );
-  newurl.setPathName( ( Pathname("./"+_url.getPathName()) / filename_r ).asString().substr(1) );
-  return newurl;
-}
-
 ///////////////////////////////////////////////////////////////////
 
 void MediaCurl::getFile( const OnMediaLocation &file ) const

diff --git a/zypp/media/MediaCurl.h b/zypp/media/MediaCurl.h
@@ -126,11 +126,6 @@ class MediaCurl : public MediaNetworkCommonHandler
      * \throws MediaCurlSetOptException if there is a problem
      **/
     virtual void setupEasy();
-    /**
-     * concatenate the attach url and the filename to a complete
-     * download url
-     **/
-    Url getFileUrl(const Pathname & filename) const;
 
     /**
      * Evaluates a curl return code and throws the right MediaException

diff --git a/zypp/media/MediaHandler.h b/zypp/media/MediaHandler.h
@@ -427,7 +427,7 @@ class MediaHandler {
          * \throws MediaException
          *
          **/
-        virtual bool getDoesFileExist( const Pathname & filename ) const = 0;
+        virtual bool getDoesFileExist( const Pathname & filename ) const =   0;
 
   protected:
 
@@ -717,5 +717,3 @@ class MediaHandler {
 
 
 #endif // ZYPP_MEDIA_MEDIAHANDLERL_H
-
-