Shadow for buildings (#668)

Co-authored-by: Izumi Kawashima <schedul.xor@gmail.com>

vtm/resources/assets/shaders/extrusion_layer_ext.glsl

@@ -13,15 +13,31 @@ varying vec4 color;
 //varying float depth;
 const float ff = 255.0;
 
+#ifdef SHADOW
+uniform mat4 u_light_mvp;
+varying vec4 v_shadow_coords;
+#endif
+
 /**
  * The diffuse of surface dependent on the light position.
  *
  * @param r_norm the normal vector of vertex's face
  */
-float diffuse(in vec3 r_norm) {
+float diffuse(in vec3 r_norm, out bool hasLight) {
     float l = dot(normalize(r_norm), normalize(u_light));
+    hasLight = l > 0.0;
     l = clamp((1.0 + l) / 2.0, 0.0, 1.0);
-    return(0.8 + l * 0.2);
+    #ifdef SHADOW
+    if (hasLight) {
+        //l = (l + (1.0 - r_norm.z)) * 0.5;
+        l = 0.8 + l * 0.2;
+    } else {
+        l = 0.5 + l * 0.3;
+    }
+    #else
+    l = 0.8 + l * 0.2;
+    #endif
+    return (l);
 }
 
 void main() {
@@ -31,6 +47,8 @@ void main() {
         height = u_zlimit;
     }
     gl_Position = u_mvp * vec4(a_pos.xy, height, 1.0);
+    bool hasLight = false;
+
     //depth = gl_Position.z;
     if (u_mode == -1) {
         ;
@@ -40,7 +58,7 @@ void main() {
             // roof / depth pass
             r_norm = vec3(0.0, 0.0, 1.0);
             color = u_color[0] * u_alpha;
-            color.rgb *= diffuse(r_norm);
+            color.rgb *= diffuse(r_norm, hasLight);
         } else {
             float lightX = u_mode == 1 ? a_normal.y : a_normal.x;
             r_norm.x = (lightX / ff) * 2.0 - 1.0;
@@ -59,7 +77,7 @@ void main() {
                 // sides 2 - use 0x00ff
                 color = u_color[2];
             }
-            color.rgb *= diffuse(r_norm) * z * h;
+            color.rgb *= diffuse(r_norm, hasLight) * z * h;
         }
         color *= u_alpha;
     } else if (u_mode == 3) {
@@ -67,6 +85,15 @@ void main() {
         float z = (0.98 - gl_Position.z * 0.02);
         color = u_color[3] * z;
     }
+    #ifdef SHADOW
+    if (hasLight) {
+        vec4 positionFromLight = u_light_mvp * a_pos;
+        v_shadow_coords = (positionFromLight / positionFromLight.w);
+    } else {
+        // Discard shadow on unlighted faces
+        v_shadow_coords = vec4(-1.0);
+    }
+    #endif
 }
 
 $$
@@ -76,6 +103,76 @@ precision highp float;
 #endif
 varying vec4 color;
 
-void main() {
-    gl_FragColor = color;
+#ifdef SHADOW
+varying vec4 v_shadow_coords; // the coords in shadow map
+
+uniform sampler2D u_shadowMap;
+uniform vec4 u_lightColor;
+uniform float u_shadowRes;
+
+const bool DEBUG = false;
+
+const float transitionDistance = 0.05; // relative transition distance at the border of shadow tex
+const float minTrans = 1.0 - transitionDistance;
+
+const int pcfCount = 2; // the number of surrounding pixels to smooth shadow
+const float biasOffset = 0.005; // offset to remove shadow acne
+const float pcfTexels = float((pcfCount * 2 + 1) * (pcfCount * 2 + 1));
+
+#if GLVERSION == 20
+float decodeFloat (vec4 color) {
+    const vec4 bitShift = vec4(
+    1.0 / (256.0 * 256.0 * 256.0),
+    1.0 / (256.0 * 256.0),
+    1.0 / 256.0,
+    1.0
+    );
+    return dot(color, bitShift);
+}
+#endif
+#endif
+
+void main() {
+    #ifdef SHADOW
+    float shadowX = abs((v_shadow_coords.x - 0.5) * 2.0);
+    float shadowY = abs((v_shadow_coords.y - 0.5) * 2.0);
+    if (shadowX > 1.0 || shadowY > 1.0) {
+        // Outside the light texture set to 0.0
+        gl_FragColor = vec4(color.rgb * u_lightColor.rgb, color.a);
+        if (DEBUG) {
+            gl_FragColor = vec4(0.0, 1.0, 0.0, 0.1);
+        }
+    } else {
+        // Inside set to 1.0; make a transition to the border
+        float shadowOpacity = (shadowX < minTrans && shadowY < minTrans) ? 1.0 :
+        (1.0 - (max(shadowX - minTrans, shadowY - minTrans) / transitionDistance));
+        float distanceToLight = clamp(v_shadow_coords.z - biasOffset, 0.0, 1.0); // avoid unexpected shadow
+
+        // Smooth shadow at borders
+        float shadowDiffuse = 0.0;
+        float texelSize = 1.0 / u_shadowRes;
+        for (int x = -pcfCount; x <= pcfCount; x++) {
+            for (int y = -pcfCount; y <= pcfCount; y++) {
+                #if GLVERSION == 30
+                float depth = texture2D(u_shadowMap, v_shadow_coords.xy + vec2(x, y) * texelSize).r;
+                #else
+                float depth = decodeFloat(texture2D(u_shadowMap, v_shadow_coords.xy + vec2(x, y) * texelSize));
+                #endif
+                if (distanceToLight > depth) {
+                    shadowDiffuse += 1.0;
+                }
+            }
+        }
+        shadowDiffuse /= pcfTexels;
+        shadowDiffuse *= shadowOpacity;
+
+        if (DEBUG && shadowDiffuse < 1.0) {
+            gl_FragColor = vec4(shadowDiffuse, color.gb, 0.1);
+        } else {
+            gl_FragColor = vec4((color.rgb * u_lightColor.rgb) * (1.0 - u_lightColor.a * shadowDiffuse), color.a);
+        }
+    }
+    #else
+    gl_FragColor = color;
+    #endif
 }

vtm/resources/assets/shaders/extrusion_layer_mesh.glsl

@@ -9,6 +9,11 @@ attribute vec4 a_pos;
 attribute vec2 a_normal;
 varying vec4 color;
 
+#ifdef SHADOW
+uniform mat4 u_light_mvp;
+varying vec4 v_shadow_coords;
+#endif
+
 void main() {
     // change height by u_alpha
     vec4 pos = a_pos;
@@ -28,18 +33,40 @@ void main() {
 
     float l = dot(r_norm, normalize(u_light));
 
-    //l *= 0.8
+    #ifdef SHADOW
+    bool hasLight = l > 0.0;
+    #endif
+
+    //l *= 0.8;
     //vec3 opp_light_dir = normalize(vec3(-u_light.xy, u_light.z));
     //l += dot(r_norm, opp_light_dir) * 0.2;
 
     // [-1,1] to range [0,1]
     l = (1.0 + l) / 2.0;
 
+    #ifdef SHADOW
+    if (hasLight) {
+        l = 0.75 + l * 0.25;
+    } else {
+        l = 0.5 + l * 0.3;
+    }
+    #else
     l = 0.75 + l * 0.25;
+    #endif
 
     // extreme fake-ssao by height
     l += (clamp(a_pos.z / 2048.0, 0.0, 0.1) - 0.05);
     color = vec4(u_color.rgb * (clamp(l, 0.0, 1.0)), u_color.a) * u_alpha;
+
+    #ifdef SHADOW
+    if (hasLight) {
+        vec4 positionFromLight = u_light_mvp * a_pos;
+        v_shadow_coords = (positionFromLight / positionFromLight.w);
+    } else {
+        // Discard shadow on unlighted faces
+        v_shadow_coords = vec4(-1.0);
+    }
+    #endif
 }
 
 $$
@@ -49,6 +76,77 @@ precision highp float;
 #endif
 varying vec4 color;
 
-void main() {
-    gl_FragColor = color;
+#ifdef SHADOW
+varying vec4 v_shadow_coords; // the coords in shadow map
+
+uniform sampler2D u_shadowMap;
+uniform vec4 u_lightColor;
+uniform float u_shadowRes;
+uniform int u_mode;
+
+const bool DEBUG = false;
+
+const float transitionDistance = 0.05; // relative transition distance at the border of shadow tex
+const float minTrans = 1.0 - transitionDistance;
+
+const int pcfCount = 2; // the number of surrounding pixels to smooth shadow
+const float biasOffset = 0.005; // offset to remove shadow acne
+const float pcfTexels = float((pcfCount * 2 + 1) * (pcfCount * 2 + 1));
+
+#if GLVERSION == 20
+float decodeFloat (vec4 color) {
+    const vec4 bitShift = vec4(
+    1.0 / (256.0 * 256.0 * 256.0),
+    1.0 / (256.0 * 256.0),
+    1.0 / 256.0,
+    1.0
+    );
+    return dot(color, bitShift);
+}
+#endif
+#endif
+
+void main() {
+    #ifdef SHADOW
+    float shadowX = abs((v_shadow_coords.x - 0.5) * 2.0);
+    float shadowY = abs((v_shadow_coords.y - 0.5) * 2.0);
+    if (shadowX > 1.0 || shadowY > 1.0) {
+        // Outside the light texture set to 0.0
+        gl_FragColor = vec4(color.rgb * u_lightColor.rgb, color.a);
+        if (DEBUG) {
+            gl_FragColor = vec4(0.0, 1.0, 0.0, 0.1);
+        }
+    } else {
+        // Inside set to 1.0; make a transition to the border
+        float shadowOpacity = (shadowX < minTrans && shadowY < minTrans) ? 1.0 :
+        (1.0 - (max(shadowX - minTrans, shadowY - minTrans) / transitionDistance));
+        float distanceToLight = clamp(v_shadow_coords.z - biasOffset, 0.0, 1.0); // avoid unexpected shadow
+
+        // smooth shadow at borders
+        float shadowDiffuse = 0.0;
+        float texelSize = 1.0 / u_shadowRes;
+        for (int x = -pcfCount; x <= pcfCount; x++) {
+            for (int y = -pcfCount; y <= pcfCount; y++) {
+                #if GLVERSION == 30
+                float depth = texture2D(u_shadowMap, v_shadow_coords.xy + vec2(x, y) * texelSize).r;
+                #else
+                float depth = decodeFloat(texture2D(u_shadowMap, v_shadow_coords.xy + vec2(x, y) * texelSize));
+                #endif
+                if (distanceToLight > depth) {
+                    shadowDiffuse += 1.0;
+                }
+            }
+        }
+        shadowDiffuse /= pcfTexels;
+        shadowDiffuse *= shadowOpacity;
+
+        if (DEBUG && shadowDiffuse < 1.0) {
+            gl_FragColor = vec4(shadowDiffuse, color.gb, 0.1);
+        } else {
+            gl_FragColor = vec4((color.rgb * u_lightColor.rgb) * (1.0 - u_lightColor.a * shadowDiffuse), color.a);
+        }
+    }
+    #else
+    gl_FragColor = color;
+    #endif
 }

vtm/resources/assets/shaders/extrusion_shadow_ground.glsl

@@ -0,0 +1,91 @@
+#ifdef GLES
+precision highp float;
+#endif
+attribute vec4 a_pos;
+uniform mat4 u_mvp;
+uniform mat4 u_light_mvp;
+
+varying vec4 v_shadow_coords;
+
+void main(void) {
+    gl_Position = u_mvp * a_pos;
+    vec4 positionFromLight = u_light_mvp * a_pos;
+    v_shadow_coords = (positionFromLight / positionFromLight.w);
+}
+
+$$
+
+#ifdef GLES
+precision highp float;
+#endif
+varying vec4 v_shadow_coords;
+
+uniform sampler2D u_shadowMap;
+uniform vec4 u_lightColor;
+uniform float u_shadowRes;
+
+const bool DEBUG = false;
+
+const float transitionDistance = 0.05; // relative transition distance at the border of shadow tex
+const float minTrans = 1.0 - transitionDistance;
+
+const int pcfCount = 2; // the number of surrounding pixels to smooth shadow
+const float biasOffset = 0.005; // offset to remove shadow acne
+const float pcfTexels = float((pcfCount * 2 + 1) * (pcfCount * 2 + 1));
+
+#if GLVERSION == 20
+float decodeFloat (vec4 color) {
+    const vec4 bitShift = vec4(
+    1.0 / (256.0 * 256.0 * 256.0),
+    1.0 / (256.0 * 256.0),
+    1.0 / 256.0,
+    1.0
+    );
+    return dot(color, bitShift);
+}
+#endif
+
+void main() {
+    float shadowX = abs((v_shadow_coords.x - 0.5) * 2.0);
+    float shadowY = abs((v_shadow_coords.y - 0.5) * 2.0);
+    if (shadowX > 1.0 || shadowY > 1.0) {
+        // Outside the light texture set to 0.0
+        gl_FragColor = vec4(u_lightColor.rgb, 1.0);
+        if (DEBUG) {
+            gl_FragColor = vec4(0.0, 1.0, 0.0, 0.1);
+        }
+    } else {
+        // Inside set to 1.0; make a transition to the border
+        float shadowOpacity = (shadowX < minTrans && shadowY < minTrans) ? 1.0 :
+        (1.0 - (max(shadowX - minTrans, shadowY - minTrans) / transitionDistance));
+        #if GLVERSION == 30
+        float distanceToLight = v_shadow_coords.z; // remove shadow acne
+        #else
+        float distanceToLight = v_shadow_coords.z - biasOffset; // remove shadow acne
+        #endif
+        distanceToLight = clamp(distanceToLight, 0.0, 1.0); // avoid unexpected far shadow
+        // smooth shadow at borders
+        float shadowDiffuse = 0.0;
+        float texelSize = 1.0 / u_shadowRes;
+        for (int x = -pcfCount; x <= pcfCount; x++) {
+            for (int y = -pcfCount; y <= pcfCount; y++) {
+                #if GLVERSION == 30
+                float depth = texture2D(u_shadowMap, v_shadow_coords.xy + vec2(x, y) * texelSize).r;
+                #else
+                float depth = decodeFloat(texture2D(u_shadowMap, v_shadow_coords.xy + vec2(x, y) * texelSize));
+                #endif
+                if (distanceToLight > depth) {
+                    shadowDiffuse += 1.0;
+                }
+            }
+        }
+        shadowDiffuse /= pcfTexels;
+        shadowDiffuse *= shadowOpacity;
+
+        if (DEBUG && shadowDiffuse < 1.0) {
+            gl_FragColor = vec4(shadowDiffuse, 0.0, 0.0, 0.1);
+        } else {
+            gl_FragColor = vec4(u_lightColor.rgb * (1.0 - u_lightColor.a * shadowDiffuse), 1.0);
+        }
+    }
+}

vtm/resources/assets/shaders/extrusion_shadow_light.glsl

@@ -0,0 +1,53 @@
+#ifdef GLES
+precision highp float;
+precision highp int;
+#endif
+uniform mat4 u_mvp;
+uniform vec4 u_color[4];
+uniform int u_mode;
+uniform float u_zlimit;
+attribute vec4 a_pos;
+
+void main(void) {
+    gl_Position = u_mvp * a_pos;
+}
+
+$$
+
+#ifdef GLES
+precision highp float;
+precision highp int;
+#endif
+
+uniform float u_alpha;
+
+#if GLVERSION == 20
+vec4 encodeFloat (float depth) {
+    const vec4 bitShift = vec4(
+    256.0 * 256.0 * 256.0,
+    256.0 * 256.0,
+    256.0,
+    1.0
+    );
+    const vec4 bitMask = vec4(
+    0.0,
+    1.0 / 256.0,
+    1.0 / 256.0,
+    1.0 / 256.0
+    );
+    vec4 comp = fract(depth * bitShift);
+    comp -= comp.xxyz * bitMask;
+    return comp;
+}
+#endif
+
+void main(void) {
+    if (u_alpha < 0.8)
+        discard; // remove shadow when alpha is too small
+
+    #if GLVERSION == 30
+    gl_FragColor = vec4(1.0);
+    #else
+    gl_FragColor = encodeFloat(gl_FragCoord.z);
+    #endif
+}