Provide access to router within web container by hostname, fixes #842 (…

…#1151)

* Provide access to router by hostname, fixes #842
* Improve content testing with ports and https
* Earlier test shouldn't leave cruft in config.yaml; rm and start if already started

cmd/ddev/cmd/root_test.go

@@ -30,6 +30,7 @@ var (
 		HTTPProbeURI:                  "wp-admin/setup-config.php",
 		Docroot:                       "htdocs",
 		Type:                          "wordpress",
+		Safe200URIWithExpectation:     testcommon.URIWithExpect{URI: "/readme.html", Expect: "Welcome. WordPress is a very special project to me."},
 	},
 	}
 )

pkg/ddevapp/ddevapp_test.go

@@ -40,7 +40,7 @@ var (
 			DBTarURL:                      "https://github.com/drud/ddev_test_tarballs/releases/download/v1.0/wordpress_db.tar.gz",
 			Docroot:                       "htdocs",
 			Type:                          "wordpress",
-			Safe200URL:                    "/readme.html",
+			Safe200URIWithExpectation:     testcommon.URIWithExpect{URI: "/readme.html", Expect: "Welcome. WordPress is a very special project to me."},
 		},
 		{
 			Name:                          "TestPkgDrupal8",
@@ -53,7 +53,7 @@ var (
 			FullSiteTarballURL:            "",
 			Type:                          "drupal8",
 			Docroot:                       "",
-			Safe200URL:                    "/README.txt",
+			Safe200URIWithExpectation:     testcommon.URIWithExpect{URI: "/README.txt", Expect: "Drupal is an open source content management platform"},
 		},
 		{
 			Name:                          "TestPkgDrupal7", // Drupal Kickstart on D7
@@ -64,7 +64,7 @@ var (
 			FullSiteTarballURL:            "https://github.com/drud/drupal-kickstart/releases/download/v0.4.0/site.tar.gz",
 			Docroot:                       "docroot",
 			Type:                          "drupal7",
-			Safe200URL:                    "/README.txt",
+			Safe200URIWithExpectation:     testcommon.URIWithExpect{URI: "/README.txt", Expect: "Drupal is an open source content management platform"},
 			FullSiteArchiveExtPath:        "docroot/sites/default/files",
 		},
 		{
@@ -75,7 +75,7 @@ var (
 			FullSiteTarballURL:            "",
 			Docroot:                       "",
 			Type:                          "drupal6",
-			Safe200URL:                    "/CHANGELOG.txt",
+			Safe200URIWithExpectation:     testcommon.URIWithExpect{URI: "/CHANGELOG.txt", Expect: "Drupal 6.38, 2016-02-24"},
 		},
 		{
 			Name:                          "TestPkgBackdrop",
@@ -85,7 +85,7 @@ var (
 			FullSiteTarballURL:            "",
 			Docroot:                       "",
 			Type:                          "backdrop",
-			Safe200URL:                    "/README.md",
+			Safe200URIWithExpectation:     testcommon.URIWithExpect{URI: "/README.md", Expect: "Backdrop is a full-featured content management system"},
 		},
 		{
 			Name:                          "TestPkgTypo3",
@@ -95,7 +95,7 @@ var (
 			FullSiteTarballURL:            "",
 			Docroot:                       "",
 			Type:                          "typo3",
-			Safe200URL:                    "/INSTALL.md",
+			Safe200URIWithExpectation:     testcommon.URIWithExpect{URI: "/INSTALL.md", Expect: "TYPO3 is an open source PHP based web content management"},
 		},
 	}
 	FullTestSites = TestSites
@@ -297,16 +297,10 @@ func TestDdevStartMultipleHostnames(t *testing.T) {
 			assert.True(check, "Container check on %s failed", containerType)
 		}
 
-		dockerIP, err := dockerutil.GetDockerIP()
-		assert.NoError(err)
 		for _, hostname := range app.GetHostnames() {
-			o := util.NewHTTPOptions("http://" + dockerIP + site.Safe200URL)
-			o.ExpectedStatus = 200
-			o.Timeout = 5
-			o.TickerInterval = 1
-			o.Headers["Host"] = hostname
-			err = util.EnsureHTTPStatus(o)
-			assert.NoError(err)
+			testcommon.EnsureLocalHTTPContent(t, "http://"+hostname+site.Safe200URIWithExpectation.URI, site.Safe200URIWithExpectation.Expect)
+			testcommon.EnsureLocalHTTPContent(t, "https://"+hostname+site.Safe200URIWithExpectation.URI, site.Safe200URIWithExpectation.Expect)
+
 		}
 
 		// Multiple projects can't run at the same time with the fqdns, so we need to clean
@@ -316,9 +310,7 @@ func TestDdevStartMultipleHostnames(t *testing.T) {
 		err = app.WriteConfig()
 		assert.NoError(err)
 
-		err = app.Stop()
-		assert.NoError(err)
-		err = app.Down(false, false)
+		err = app.Down(true, false)
 		assert.NoError(err)
 
 		runTime()
@@ -1527,6 +1519,13 @@ func TestGetAllURLs(t *testing.T) {
 
 		assert.True(exists, "URL list for app: %s does not contain direct web container address: %s", app.Name, expectedDirectAddress)
 
+		// Multiple projects can't run at the same time with the fqdns, so we need to clean
+		// up these for tests that run later.
+		app.AdditionalFQDNs = []string{}
+		app.AdditionalHostnames = []string{}
+		err = app.WriteConfig()
+		assert.NoError(err)
+
 		err = app.Stop()
 		assert.NoError(err)
 
@@ -1665,6 +1664,60 @@ func TestDbMigration(t *testing.T) {
 	switchDir()
 }
 
+// TestInternalAndExternalAccessToURL checks we can access content from host and from inside container by URL (with port)
+func TestInternalAndExternalAccessToURL(t *testing.T) {
+	assert := asrt.New(t)
+
+	for _, site := range TestSites {
+		runTime := testcommon.TimeTrack(time.Now(), fmt.Sprintf("%s TestInternalAndExternalAccessToURL", site.Name))
+
+		app := new(ddevapp.DdevApp)
+
+		err := app.Init(site.Dir)
+		assert.NoError(err)
+
+		//nolint: vet
+		for _, pair := range []testcommon.PortPair{{"80", "443"}, {"8080", "8443"}} {
+			testcommon.ClearDockerEnv()
+			app.RouterHTTPPort = pair.HTTPPort
+			app.RouterHTTPSPort = pair.HTTPSPort
+			err = app.WriteConfig()
+			assert.NoError(err)
+
+			if app.SiteStatus() == ddevapp.SiteStopped || app.SiteStatus() == ddevapp.SiteRunning {
+				err = app.Down(true, false)
+				assert.NoError(err)
+			}
+			err = app.Start()
+			assert.NoError(err)
+
+			// Ensure that we can access from the host even with extra port specifications.
+			testcommon.EnsureLocalHTTPContent(t, app.GetHTTPURL()+site.Safe200URIWithExpectation.URI, site.Safe200URIWithExpectation.Expect)
+			testcommon.EnsureLocalHTTPContent(t, app.GetHTTPSURL()+site.Safe200URIWithExpectation.URI, site.Safe200URIWithExpectation.Expect)
+
+			// Ensure that we can access the same URL from within the web container (via router)
+			var out string
+			out, _, err = app.Exec("web", "curl", "-sk", app.GetHTTPURL()+site.Safe200URIWithExpectation.URI)
+			assert.NoError(err)
+			assert.Contains(out, site.Safe200URIWithExpectation.Expect)
+
+			out, _, err = app.Exec("web", "curl", "-sk", app.GetHTTPSURL()+site.Safe200URIWithExpectation.URI)
+			assert.NoError(err)
+			assert.Contains(out, site.Safe200URIWithExpectation.Expect)
+		}
+
+		// Set the ports back to the default was so we don't break any following tests.
+		app.RouterHTTPSPort = "443"
+		app.RouterHTTPPort = "80"
+		err = app.WriteConfig()
+		assert.NoError(err)
+		err = app.Stop()
+		assert.NoError(err)
+
+		runTime()
+	}
+}
+
 // constructContainerName builds a container name given the type (web/db/dba) and the app
 func constructContainerName(containerType string, app *ddevapp.DdevApp) (string, error) {
 	container, err := app.FindContainerByType(containerType)

pkg/ddevapp/templates.go

@@ -78,6 +78,9 @@ services:
       com.ddev.approot: $DDEV_APPROOT
       com.ddev.app-url: $DDEV_URL
     extra_hosts: ["{{ .extra_host }}"]
+    external_links:
+      - ddev-router:$DDEV_HOSTNAME
+
   dba:
     container_name: ddev-${DDEV_SITENAME}-dba
     image: $DDEV_DBAIMAGE

pkg/testcommon/testcommon.go

@@ -28,6 +28,12 @@ import (
 	"testing"
 )
 
+// URIWithExpect pairs a URI like "/readme.html" with some substring content "should be found in URI"
+type URIWithExpect struct {
+	URI    string
+	Expect string
+}
+
 // TestSite describes a site for testing, with name, URL of tarball, and optional dir.
 type TestSite struct {
 	// Name is the generic name of the site, and is used as the default dir.
@@ -55,9 +61,8 @@ type TestSite struct {
 	// Type is the type of application. This can be specified when a config file is not present
 	// for a test site.
 	Type string
-	// Safe200URL is a string of a url that can be accessed for testing a site
-	// that has not yet been installed
-	Safe200URL string
+	// Safe200URIWithExpectation provides a static URI with contents that it can be expected to contain.
+	Safe200URIWithExpectation URIWithExpect
 	// FullSiteArchiveExtPath is the path that should be extracted from inside an archive when
 	// importing the files from a full site archive
 	FullSiteArchiveExtPath string
@@ -351,12 +356,18 @@ func GetLocalHTTPResponse(t *testing.T, rawurl string) (string, *http.Response,
 	if err != nil {
 		t.Fatalf("Failed to parse url %s: %v", rawurl, err)
 	}
+	port := u.Port()
 
 	dockerIP, err := dockerutil.GetDockerIP()
 	assert.NoError(err)
 
 	fakeHost := u.Hostname()
 	u.Host = dockerIP
+	// Add the port if there is one.
+	if port != "" {
+		fakeHost = u.Hostname() + ":" + port
+		u.Host = dockerIP + ":" + port
+	}
 	localAddress := u.String()
 
 	timeout := time.Duration(10 * time.Second)
@@ -408,3 +419,9 @@ func EnsureLocalHTTPContent(t *testing.T, rawurl string, expectedContent string)
 	assert.NoError(err, "GetLocalHTTPResponse returned err on rawurl %s: %v", rawurl, err)
 	assert.Contains(body, expectedContent)
 }
+
+// PortPair is for tests to use naming portsets for tests
+type PortPair struct {
+	HTTPPort  string
+	HTTPSPort string
+}

pkg/testcommon/testcommon_test.go

@@ -23,7 +23,7 @@ var TestSites = []TestSite{
 		DBTarURL:                      "https://github.com/drud/wordpress/releases/download/v0.4.0/db.tar.gz",
 		Docroot:                       "htdocs",
 		Type:                          "wordpress",
-		Safe200URL:                    "/readme.html",
+		Safe200URIWithExpectation:     URIWithExpect{URI: "/readme.html", Expect: "Welcome. WordPress is a very special project to me."},
 	},
 }
 
@@ -166,16 +166,36 @@ func TestGetLocalHTTPResponse(t *testing.T) {
 	err := app.Init(site.Dir)
 	assert.NoError(err)
 
-	err = app.Start()
-	assert.NoError(err)
-
-	safeURL := app.GetHTTPURL() + site.Safe200URL
-	out, _, err := GetLocalHTTPResponse(t, safeURL)
+	for _, pair := range []PortPair{{"80", "443"}, {"8080", "8443"}} {
+		ClearDockerEnv()
+		app.RouterHTTPPort = pair.HTTPPort
+		app.RouterHTTPSPort = pair.HTTPSPort
+		err = app.WriteConfig()
+		assert.NoError(err)
+
+		err = app.Start()
+		assert.NoError(err)
+
+		safeURL := app.GetHTTPURL() + site.Safe200URIWithExpectation.URI
+		//nolint: vetshadow
+		out, _, err := GetLocalHTTPResponse(t, safeURL)
+		assert.NoError(err)
+		assert.Contains(out, site.Safe200URIWithExpectation.Expect)
+
+		safeURL = app.GetHTTPSURL() + site.Safe200URIWithExpectation.URI
+		out, _, err = GetLocalHTTPResponse(t, safeURL)
+		assert.NoError(err)
+		assert.Contains(out, site.Safe200URIWithExpectation.Expect)
+
+		// This does the same thing as previous, but worth exercising it here.
+		EnsureLocalHTTPContent(t, safeURL, site.Safe200URIWithExpectation.Expect)
+	}
+	// Set the ports back to the default was so we don't break any following tests.
+	app.RouterHTTPSPort = "443"
+	app.RouterHTTPPort = "80"
+	err = app.WriteConfig()
 	assert.NoError(err)
-	assert.Contains(out, "Famous 5-minute install")
 
-	// This does the same thing as previous, but worth exercising it here.
-	EnsureLocalHTTPContent(t, safeURL, "Famous 5-minute install")
 	err = app.Down(true, false)
 	assert.NoError(err)