Dockerize webcrawler (#66)

* dockerized webcrawler

* functional docker-compose webcrawler

* made it docker-compose can auto-scale tracker, crawler, and lighthouse roles

Cluster.WebCrawler/.dockerignore

@@ -0,0 +1,12 @@
+.dockerignore
+.env
+.git
+.gitignore
+.vs
+.vscode
+docker-compose.yml
+docker-compose.*.yml
+*/bin
+*/obj
+!obj/Docker/publish/*
+!obj/Docker/empty/

Cluster.WebCrawler/WebCrawler.sln

@@ -13,6 +13,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "WebCrawler.CrawlService", "
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "WebCrawler.Web", "src\WebCrawler.Web\WebCrawler.Web.csproj", "{297D37F3-5CAA-4B4F-A6EA-0A76AFDDE80C}"
 EndProject
+Project("{E53339B2-1760-4266-BCC7-CA923CBCF16C}") = "docker-compose", "docker-compose.dcproj", "{188B87E9-7FF0-4488-A340-FE8A62688B27}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -39,6 +41,10 @@ Global
 		{297D37F3-5CAA-4B4F-A6EA-0A76AFDDE80C}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{297D37F3-5CAA-4B4F-A6EA-0A76AFDDE80C}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{297D37F3-5CAA-4B4F-A6EA-0A76AFDDE80C}.Release|Any CPU.Build.0 = Release|Any CPU
+		{188B87E9-7FF0-4488-A340-FE8A62688B27}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{188B87E9-7FF0-4488-A340-FE8A62688B27}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{188B87E9-7FF0-4488-A340-FE8A62688B27}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{188B87E9-7FF0-4488-A340-FE8A62688B27}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

Cluster.WebCrawler/docker-compose.ci.build.yml

@@ -0,0 +1,9 @@
+version: '3'
+
+services:
+  ci-build:
+    image: microsoft/aspnetcore-build:1.0-2.0
+    volumes:
+      - .:/src
+    working_dir: /src
+    command: /bin/bash -c "dotnet restore ./WebCrawler.sln && dotnet publish ./WebCrawler.sln -c Release -o ./obj/Docker/publish"

Cluster.WebCrawler/docker-compose.dcproj

@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="15.0" Sdk="Microsoft.Docker.Sdk">
+  <PropertyGroup Label="Globals">
+    <ProjectVersion>2.0</ProjectVersion>
+    <DockerTargetOS>Linux</DockerTargetOS>
+    <ProjectGuid>188b87e9-7ff0-4488-a340-fe8a62688b27</ProjectGuid>
+    <DockerLaunchBrowser>True</DockerLaunchBrowser>
+    <DockerServiceUrl>http://localhost:{ServicePort}</DockerServiceUrl>
+    <DockerServiceName>webcrawler.web</DockerServiceName>
+  </PropertyGroup>
+  <ItemGroup>
+    <None Include="docker-compose.override.yml">
+      <DependentUpon>docker-compose.yml</DependentUpon>
+    </None>
+    <None Include="docker-compose.yml" />
+    <None Include=".dockerignore" />
+    <None Include="docker-compose.ci.build.yml" />
+  </ItemGroup>
+</Project>

Cluster.WebCrawler/docker-compose.override.yml

@@ -0,0 +1,8 @@
+version: '3'
+
+services:
+  webcrawler.web:
+    environment:
+      - ASPNETCORE_ENVIRONMENT=Development
+    ports:
+      - "80"

Cluster.WebCrawler/docker-compose.yml

@@ -0,0 +1,52 @@
+version: '3'
+
+services:
+  lighthouse:
+    image: petabridge/lighthouse:0.9.1
+    ports:
+      - '9110:9110'
+      - '4053:4053'
+    environment:
+      ACTORSYSTEM: "webcrawler"
+      CLUSTER_IP: lighthouse
+      CLUSTER_PORT: 4053
+      CLUSTER_SEEDS: "akka.tcp://webcrawler@lighthouse:4053"
+
+
+  webcrawler.web:
+    image: webcrawler.web
+    build:
+      context: .
+      dockerfile: src/WebCrawler.Web/Dockerfile
+    ports:
+      - '8080:80'
+    environment:
+      CLUSTER_SEEDS: "akka.tcp://webcrawler@lighthouse:4053"
+      CLUSTER_IP: webcrawler.web
+      CLUSTER_PORT: 0
+    depends_on:
+      - "lighthouse"
+
+  webcrawler.crawlservice:
+    image: webcrawler.crawlservice
+    build:
+      context: .
+      dockerfile: src/WebCrawler.CrawlService/Dockerfile
+    environment:
+      CLUSTER_SEEDS: "akka.tcp://webcrawler@lighthouse:4053"
+      CLUSTER_IP: webcrawler.crawlservice
+      CLUSTER_PORT: 0
+    depends_on:
+      - "lighthouse"
+
+  webcrawler.trackerservice:
+    image: webcrawler.trackerservice
+    build:
+      context: .
+      dockerfile: src/WebCrawler.TrackerService/Dockerfile
+    environment:
+      CLUSTER_SEEDS: "akka.tcp://webcrawler@lighthouse:4053"
+      CLUSTER_IP: webcrawler.trackerservice
+      CLUSTER_PORT: 0
+    depends_on:
+      - "lighthouse"

Cluster.WebCrawler/get-dockerip.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+if [ -z "$CLUSTER_IP"]; then
+	host=$(hostname -i)
+	echo "Docker container bound on $host"
+	export CLUSTER_IP="$host"
+else
+	echo "Docker container bound on $CLUSTER_IP"
+fi
+
+exec "$@"

Cluster.WebCrawler/src/WebCrawler.CrawlService/CrawlerService.cs

@@ -3,6 +3,7 @@
 using System.Text;
 using System.Threading.Tasks;
 using Akka.Actor;
+using Akka.Bootstrap.Docker;
 using WebCrawler.Shared.Config;
 
 namespace WebCrawler.CrawlService
@@ -17,7 +18,7 @@ public class CrawlerService
         public bool Start()
         {
             var config = HoconLoader.ParseConfig("crawler.hocon");
-            ClusterSystem = ActorSystem.Create("webcrawler", config);
+            ClusterSystem = ActorSystem.Create("webcrawler", config.BootstrapFromDocker());
             return true;
         }
 

Cluster.WebCrawler/src/WebCrawler.CrawlService/Dockerfile

@@ -0,0 +1,34 @@
+FROM microsoft/dotnet:2.0-runtime AS base
+WORKDIR /app
+
+# should be a comma-delimited list
+ENV CLUSTER_SEEDS "[]"
+ENV CLUSTER_IP ""
+ENV CLUSTER_PORT "5213"
+
+#Akka.Remote inbound listening endpoint
+EXPOSE 5213 
+
+FROM microsoft/dotnet:2.0-sdk AS build
+WORKDIR /src
+COPY *.sln ./
+COPY ./get-dockerip.sh ./get-dockerip.sh
+COPY src/WebCrawler.CrawlService/WebCrawler.CrawlService.csproj src/WebCrawler.CrawlService/
+COPY src/WebCrawler.Shared/WebCrawler.Shared.csproj src/WebCrawler.Shared/
+COPY src/WebCrawler.Shared.IO/WebCrawler.Shared.IO.csproj src/WebCrawler.Shared.IO/
+RUN dotnet restore
+COPY . .
+WORKDIR /src/src/WebCrawler.CrawlService
+RUN dotnet build -c Release -o /app
+
+FROM build AS publish
+RUN dotnet publish -c Release -o /app
+
+FROM base AS final
+WORKDIR /app
+COPY --from=build /src/get-dockerip.sh ./get-dockerip.sh
+COPY --from=publish /app .
+
+ENTRYPOINT ["/bin/bash","get-dockerip.sh"]
+
+CMD ["dotnet", "WebCrawler.CrawlService.dll"]

Cluster.WebCrawler/src/WebCrawler.CrawlService/WebCrawler.CrawlService.csproj

@@ -16,6 +16,7 @@
   </ItemGroup>
 
   <ItemGroup>
+    <PackageReference Include="Akka.Bootstrap.Docker" Version="0.1.0" />
     <PackageReference Include="Akka.Cluster" Version="1.3.5" />
   </ItemGroup>
 

Cluster.WebCrawler/src/WebCrawler.TrackerService/Dockerfile

@@ -0,0 +1,34 @@
+FROM microsoft/dotnet:2.0-runtime AS base
+WORKDIR /app
+
+# should be a comma-delimited list
+ENV CLUSTER_SEEDS "[]"
+ENV CLUSTER_IP ""
+ENV CLUSTER_PORT "5212"
+
+#Akka.Remote inbound listening endpoint
+EXPOSE 5212 
+
+FROM microsoft/dotnet:2.0-sdk AS build
+WORKDIR /src
+COPY *.sln ./
+COPY ./get-dockerip.sh ./get-dockerip.sh
+COPY src/WebCrawler.TrackerService/WebCrawler.TrackerService.csproj src/WebCrawler.TrackerService/
+COPY src/WebCrawler.Shared/WebCrawler.Shared.csproj src/WebCrawler.Shared/
+COPY src/WebCrawler.Shared.IO/WebCrawler.Shared.IO.csproj src/WebCrawler.Shared.IO/
+RUN dotnet restore
+COPY . .
+WORKDIR /src/src/WebCrawler.TrackerService
+RUN dotnet build -c Release -o /app
+
+FROM build AS publish
+RUN dotnet publish -c Release -o /app
+
+FROM base AS final
+WORKDIR /app
+COPY --from=build /src/get-dockerip.sh ./get-dockerip.sh
+COPY --from=publish /app .
+
+ENTRYPOINT ["/bin/bash","get-dockerip.sh"]
+
+CMD ["dotnet", "WebCrawler.TrackerService.dll"]

Cluster.WebCrawler/src/WebCrawler.TrackerService/TrackerService.cs

@@ -6,6 +6,7 @@
 
 using System.Threading.Tasks;
 using Akka.Actor;
+using Akka.Bootstrap.Docker;
 using WebCrawler.Shared.Config;
 using WebCrawler.TrackerService.Actors;
 using WebCrawler.TrackerService.Actors.Tracking;
@@ -24,7 +25,7 @@ public class TrackerService
         public bool Start()
         {
             var config = HoconLoader.ParseConfig("tracker.hocon");
-            ClusterSystem = ActorSystem.Create("webcrawler", config);
+            ClusterSystem = ActorSystem.Create("webcrawler", config.BootstrapFromDocker());
             ApiMaster = ClusterSystem.ActorOf(Props.Create(() => new ApiMaster()), "api");
             DownloadMaster = ClusterSystem.ActorOf(Props.Create(() => new DownloadsMaster()), "downloads");
             return true;

Cluster.WebCrawler/src/WebCrawler.TrackerService/WebCrawler.TrackerService.csproj

@@ -16,6 +16,7 @@
   </ItemGroup>
 
   <ItemGroup>
+    <PackageReference Include="Akka.Bootstrap.Docker" Version="0.1.0" />
     <PackageReference Include="Akka.Cluster" Version="1.3.5" />
   </ItemGroup>
 

Cluster.WebCrawler/src/WebCrawler.Web/AkkaStartupTasks.cs

@@ -1,4 +1,5 @@
 using Akka.Actor;
+using Akka.Bootstrap.Docker;
 using Akka.Routing;
 using WebCrawler.Shared.Config;
 using WebCrawler.Web.Actors;
@@ -10,7 +11,7 @@ public static class AkkaStartupTasks
         public static ActorSystem StartAkka()
         {
             var config = HoconLoader.ParseConfig("web.hocon");
-            SystemActors.ActorSystem = ActorSystem.Create("webcrawler", config);
+            SystemActors.ActorSystem = ActorSystem.Create("webcrawler", config.BootstrapFromDocker());
             var router = SystemActors.ActorSystem.ActorOf(Props.Empty.WithRouter(FromConfig.Instance), "tasker");
             var processor = SystemActors.CommandProcessor = SystemActors.ActorSystem.ActorOf(Props.Create(() => new CommandProcessor(router)),
                 "commands");

Cluster.WebCrawler/src/WebCrawler.Web/Dockerfile

@@ -0,0 +1,36 @@
+FROM microsoft/aspnetcore:2.0 AS base
+WORKDIR /app
+
+# should be a comma-delimited list
+ENV CLUSTER_SEEDS "[]"
+ENV CLUSTER_IP ""
+ENV CLUSTER_PORT "16666"
+
+EXPOSE 80
+
+#Akka.Remote inbound listening endpoint
+EXPOSE 16666 
+
+
+FROM microsoft/aspnetcore-build:2.0 AS build
+WORKDIR /src
+COPY *.sln ./
+COPY ./get-dockerip.sh ./get-dockerip.sh
+COPY src/WebCrawler.Web/WebCrawler.Web.csproj src/WebCrawler.Web/
+COPY src/WebCrawler.Shared/WebCrawler.Shared.csproj src/WebCrawler.Shared/
+RUN dotnet restore
+COPY . .
+WORKDIR /src/src/WebCrawler.Web
+RUN dotnet build -c Release -o /app
+
+FROM build AS publish
+RUN dotnet publish -c Release -o /app
+
+FROM base AS final
+WORKDIR /app
+COPY --from=build /src/get-dockerip.sh ./get-dockerip.sh
+COPY --from=publish /app .
+
+ENTRYPOINT ["/bin/bash","get-dockerip.sh"]
+
+CMD ["dotnet", "WebCrawler.Web.dll"]

Cluster.WebCrawler/src/WebCrawler.Web/WebCrawler.Web.csproj

@@ -1,8 +1,18 @@
 <Project Sdk="Microsoft.NET.Sdk.Web">
   <PropertyGroup>
     <TargetFramework>netcoreapp2.0</TargetFramework>
+    <DockerComposeProjectPath>..\..\docker-compose.dcproj</DockerComposeProjectPath>
   </PropertyGroup>
   <ItemGroup>
+    <None Remove="web.hocon" />
+  </ItemGroup>
+  <ItemGroup>
+    <Content Include="web.hocon">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </Content>
+  </ItemGroup>
+  <ItemGroup>
+    <PackageReference Include="Akka.Bootstrap.Docker" Version="0.1.0" />
     <PackageReference Include="Akka.Cluster" Version="1.3.5" />
     <PackageReference Include="Microsoft.AspNetCore.All" Version="2.0.6" />
     <PackageReference Include="Microsoft.AspNetCore.Mvc" Version="2.0.3" />