fix the wrong normal mapping (#541)

fix wrong normal mapping

metadrive/engine/core/light.py

@@ -8,7 +8,7 @@ class Light(BaseObject):
     """
     It is dynamic element since it will follow the camera
     """
-    direction_pos = (-100, 100, 120)
+    direction_pos = (-100, 100, 100)
 
     def __init__(self):
         super(Light, self).__init__(random_seed=0)

metadrive/engine/core/terrain.py

@@ -389,7 +389,7 @@ def _load_mesh_terrain_textures(self, engine, anisotropic_degree=16, filter_type
             self.grass_rough = self.loader.loadTexture(
                 AssetLoader.file_path("textures", "grass1", "GroundGrassGreen002_BUMP_1K.jpg")
             )
-            self.grass_tex_ratio = 128.0
+            self.grass_tex_ratio = 64.0
 
         v_wrap = Texture.WMRepeat
         u_warp = Texture.WMMirror

metadrive/shaders/terrain.frag.glsl

@@ -76,16 +76,10 @@ vec3 project(mat4 mvp, vec3 p) {
 }
 
 
-vec3 get_color(vec3 diffuse, sampler2D normal_tex, sampler2D rough_tex, float tex_ratio, mat3 tbn){
-//       float roughness = texture(rough_tex, terrain_uv * tex_ratio).r;
-      vec3 normal = normalize(texture(normal_tex, terrain_uv * tex_ratio).rgb*2.0-1.0) * p3d_NormalMatrix;
-//       normal = normalize(normal + (roughness - 0.5) * 2.0);
-      vec3 basecolor = diffuse.xyz;
+vec3 get_normal(sampler2D normal_tex, sampler2D rough_tex, float tex_ratio, mat3 tbn){
+      vec3 normal = texture(normal_tex, terrain_uv * tex_ratio).rgb*2.0-1.0;
       normal = normalize(tbn * normal);
-      vec3 light_dir = normalize(light_direction);
-      vec3 shading = max(0.0, dot(normal, light_dir)) * diffuse;
-
-      return shading;
+      return normal;
 }
 
 void main() {
@@ -104,31 +98,14 @@ void main() {
   vec3 tangent = normalize(vec3(1, 0, h_u1 - h_u0));
   vec3 binormal = normalize(vec3(0, 1, h_v1 - h_v0));
   vec3 terrain_normal = normalize(cross(tangent, binormal));
-  vec3 normal = normalize(p3d_NormalMatrix * terrain_normal);
-  // normal.x *= -1;
-
-  mat3 tbn = mat3(tangent, binormal, normal);
-  vec3 shading = vec3(0.0);
+  // tbn is calulated in world space
+  mat3 tbn = mat3(tangent, binormal, terrain_normal);
 
-  // Calculate the shading of each light in the scene
-  for (int i = 0; i < p3d_LightSource.length(); ++i) {
-    vec3 diff = p3d_LightSource[i].position.xyz - vtx_pos * p3d_LightSource[i].position.w;
-    vec3 light_vector = normalize(diff);
-    vec3 light_shading = clamp(dot(normal, light_vector), 0.0, 1.0) * p3d_LightSource[i].color;
-    // If PSSM is not used, use the shadowmap from the light
-    // This is deeply ineficient, it's only to be able to compare the rendered shadows
-    if (!use_pssm) {
-      vec4 projected = projecteds[i];
-      // Apply a bias to remove some of the self-shadow acne
-      projected.z -= fixed_bias * 0.01 * projected.w;
-      light_shading *= textureProj(p3d_LightSource[i].shadowMap, projected);
-    }
-    shading += light_shading;
-  }
-  vec3 color_origin;
+  // get the color and terrain normal in world space
+  vec3 diffuse;
+  vec3 tex_normal_world;
   if ((attri.r > 0.01) && terrain_uv.x>r_min && terrain_uv.y > r_min && terrain_uv.x<r_max && terrain_uv.y<r_max){
     float value = attri.r; // Assuming it's a red channel texture
-    vec3 diffuse;
     if (value < 0.11) {
         // Semantics for value 1
         diffuse=texture(yellow_tex, terrain_uv * road_tex_ratio).rgb;
@@ -139,21 +116,41 @@ void main() {
         // Semantics for value 4
         diffuse = texture(white_tex, terrain_uv * road_tex_ratio).rgb;
     }
-    color_origin=get_color(diffuse, road_normal,  road_rough, road_tex_ratio, tbn);
+        tex_normal_world = get_normal(road_normal,  road_rough, road_tex_ratio, tbn);
   }
   else{
 
       // texture splatting, mixing ratio can be determined via rgba, no grass here
-      vec3 diffuse = texture(grass_tex, terrain_uv * grass_tex_ratio).rgb;
-      color_origin=get_color(diffuse, grass_normal, grass_rough, grass_tex_ratio, tbn);
+      diffuse = texture(grass_tex, terrain_uv * grass_tex_ratio).rgb;
+      tex_normal_world = get_normal(grass_normal, grass_rough, grass_tex_ratio, tbn);
     }
 
+//   vec3 terrain_normal_view =  normalize(tex_normal_world);
+
+  vec3 shading = vec3(0.0);
+
+  // Calculate the shading of each light in the scene
+  for (int i = 0; i < p3d_LightSource.length(); ++i) {
+    vec3 diff = p3d_LightSource[i].position.xyz - vtx_pos * p3d_LightSource[i].position.w;
+    vec3 light_vector = normalize(diff);
+    vec3 light_shading = clamp(dot(normalize(p3d_NormalMatrix * tex_normal_world), light_vector), 0.0, 1.0) * p3d_LightSource[i].color;
+    // If PSSM is not used, use the shadowmap from the light
+    // This is deeply ineficient, it's only to be able to compare the rendered shadows
+    if (!use_pssm) {
+      vec4 projected = projecteds[i];
+      // Apply a bias to remove some of the self-shadow acne
+      projected.z -= fixed_bias * 0.01 * projected.w;
+      light_shading *= textureProj(p3d_LightSource[i].shadowMap, projected);
+    }
+    shading += light_shading;
+  }
+
   // static shadow
   vec3 light_dir = normalize(light_direction);
-  shading *= max(0.0, dot(terrain_normal, light_dir));
-  shading += vec3(0.07, 0.07, 0.1);
+  shading *= max(0.0, dot(tex_normal_world, light_dir));
+//   shading += vec3(0.07, 0.07, 0.1);
 
-  // dynamic shadow
+//   dynamic shadow
   if (use_pssm) {
     // Find in which split the current point is present.
     int split = 99;
@@ -192,7 +189,7 @@ void main() {
 
   shading += p3d_LightModel.ambient.xyz;
 
-  shading *= color_origin.xyz;
+  shading *= diffuse.xyz;
 
   if (fog) {
     // Fake fog

metadrive/tests/vis_functionality/vis_rgb_cam.py

@@ -10,25 +10,24 @@
             "start_seed": 4,
             "stack_size": 5,
             "agent_policy": IDMPolicy,
-            # "debug": True,
+            "debug": True,
+            "use_mesh_terrain": True,
             # "debug_panda3d": True,
             "vehicle_config": dict(image_source="rgb_camera"),
             "sensors": {
                 "rgb_camera": (RGBCamera, 521, 512)
             },
             "interface_panel": ["dashboard", "rgb_camera"],
-            "manual_control": False,
-            "use_render": False,
+            "manual_control": True,
+            "use_render": True,
             "image_observation": True,  # it is a switch telling metadrive to use rgb as observation
             "rgb_clip": True,  # clip rgb to range(0,1) instead of (0, 255)
             # "pstats": True,
         }
     )
     env.reset()
     # # print m to capture rgb observation
-    env.engine.accept(
-        "m", env.engine.get_sensor(env.vehicle.config["image_source"]).save_image, extraArgs=[env.vehicle]
-    )
+    env.engine.accept("m", env.engine.terrain.reload_terrain_shader)
     import cv2
 
     for i in range(1, 100000):