raster-masking test combined with vector. Refactor raster overlap.

Return map z -> StencilMode instead of using callbacks.
overlapping-vector uses vector layer between two raster to guard
against side effects when refactoring stencil usage.

src/render/draw_hillshade.js

@@ -25,21 +25,19 @@ function drawHillshade(painter: Painter, sourceCache: SourceCache, layer: Hillsh
     const depthMode = painter.depthModeForSublayer(0, DepthMode.ReadOnly);
     const colorMode = painter.colorModeForRenderPass();
 
-    const [getStencilMode, done, coords] = painter.renderPass === 'translucent' ?
-        painter.setupStencilingForOverdraw(tileIDs) : [(_) => StencilMode.disabled, () => {}, tileIDs];
+    const [stencilModes, coords] = painter.renderPass === 'translucent' ?
+        painter.stencilConfigForOverlap(tileIDs) : [{}, tileIDs];
 
     for (const coord of coords) {
         const tile = sourceCache.getTile(coord);
         if (tile.needsHillshadePrepare && painter.renderPass === 'offscreen') {
             prepareHillshade(painter, tile, layer, sourceMaxZoom, depthMode, StencilMode.disabled, colorMode);
-            continue;
         } else if (painter.renderPass === 'translucent') {
-            renderHillshade(painter, tile, layer, depthMode, getStencilMode(coord), colorMode);
+            renderHillshade(painter, tile, layer, depthMode, stencilModes[coord.overscaledZ], colorMode);
         }
     }
 
     context.viewport.set([0, 0, painter.width, painter.height]);
-    done();
 }
 
 function renderHillshade(painter, tile, layer, depthMode, stencilMode, colorMode) {

src/render/draw_raster.js

@@ -28,9 +28,8 @@ function drawRaster(painter: Painter, sourceCache: SourceCache, layer: RasterSty
 
     const colorMode = painter.colorModeForRenderPass();
 
-    const [getStencilMode, done, coords] = source instanceof ImageSource ?
-        [(_) => StencilMode.disabled, () => {}, tileIDs] :
-        painter.setupStencilingForOverdraw(tileIDs);
+    const [stencilModes, coords] = source instanceof ImageSource ? [{}, tileIDs] :
+        painter.stencilConfigForOverlap(tileIDs);
 
     const minTileZ = coords[coords.length - 1].overscaledZ;
 
@@ -74,12 +73,11 @@ function drawRaster(painter: Painter, sourceCache: SourceCache, layer: RasterSty
                 uniformValues, layer.id, source.boundsBuffer,
                 painter.quadTriangleIndexBuffer, source.boundsSegments);
         } else {
-            program.draw(context, gl.TRIANGLES, depthMode, getStencilMode(coord), colorMode, CullFaceMode.disabled,
+            program.draw(context, gl.TRIANGLES, depthMode, stencilModes[coord.overscaledZ], colorMode, CullFaceMode.disabled,
                 uniformValues, layer.id, painter.rasterBoundsBuffer,
                 painter.quadTriangleIndexBuffer, painter.rasterBoundsSegments);
         }
     }
-    done();
 }
 
 function getFadeValues(tile, parentTile, sourceCache, layer, transform) {

src/render/painter.js

@@ -298,30 +298,31 @@ class Painter {
     /*
      * Sort coordinates by Z as drawing tiles is done in Z-descending order.
      * All children with the same Z write the same stencil value.  Children
-     * stencil values are greater than parent's.  Stencil ref values continue
-     * range used in _tileClippingMaskIDs.
+     * stencil values are greater than parent's.  This is used only for raster
+     * and raster-dem tiles, which are already clipped to tile boundaries, to
+     * mask area of tile overlapped by children tiles.
+     * Stencil ref values continue range used in _tileClippingMaskIDs.
      *
-     * Returns [getStencilMode(coordinate), cleanup(), sortedCoords].
+     * Returns [StencilMode for tile overscaleZ map, sortedCoords].
      */
-    setupStencilingForOverdraw(tileIDs: Array<OverscaledTileID>): [((coord: OverscaledTileID) => StencilMode), (() => void), Array<OverscaledTileID>] {
-        this.currentStencilSource = undefined;
+    stencilConfigForOverlap(tileIDs: Array<OverscaledTileID>): [{[number]: $ReadOnly<StencilMode>}, Array<OverscaledTileID>] {
         const gl = this.context.gl;
         const coords = tileIDs.sort((a, b) => b.overscaledZ - a.overscaledZ);
         const minTileZ = coords[coords.length - 1].overscaledZ;
         const stencilValues = coords[0].overscaledZ - minTileZ + 1;
-        if (this.nextStencilID + stencilValues > 256) {
-            this.clearStencil();
+        if (stencilValues > 1) {
+            this.currentStencilSource = undefined;
+            if (this.nextStencilID + stencilValues > 256) {
+                this.clearStencil();
+            }
+            const zToStencilMode = {};
+            for (let i = 0; i < stencilValues; i++) {
+                zToStencilMode[i + minTileZ] = new StencilMode({func: gl.GEQUAL, mask: 0xFF}, i + this.nextStencilID, 0xFF, gl.KEEP, gl.KEEP, gl.REPLACE);
+            }
+            this.nextStencilID += stencilValues;
+            return [zToStencilMode, coords];
         }
-        const stencilMode = new StencilMode({func: gl.GEQUAL, mask: 0xFF}, 0, 0xFF, gl.KEEP, gl.KEEP, gl.REPLACE);
-        const painter = this;
-        const getStencilMode = (coord) => {
-            stencilMode.ref = painter.nextStencilID + coord.overscaledZ - minTileZ;
-            return stencilMode;
-        };
-        const done = () => {
-            painter.nextStencilID += stencilValues;
-        };
-        return [getStencilMode, done, coords];
+        return [{[minTileZ]: StencilMode.disabled}, coords];
     }
 
     colorModeForRenderPass(): $ReadOnly<ColorMode> {

test/integration/render-tests/raster-masking/overlapping-vector/style.json

@@ -0,0 +1,90 @@
+{
+  "version": 8,
+  "metadata": {
+    "test": {
+      "height": 256
+    }
+  },
+  "center": [
+    -122.48,
+    37.84
+  ],
+  "zoom": 14,
+  "sources": {
+    "contour": {
+      "type": "raster",
+      "tiles": [
+        "local://tiles/{z}-{x}-{y}.contour.png"
+      ],
+      "maxzoom": 17,
+      "tileSize": 256
+    },
+    "geojson": {
+      "type": "geojson",
+      "data": {
+        "type": "Polygon",
+        "coordinates": [
+          [
+            [
+              -123,
+              37.839
+            ],
+            [
+              -123,
+              37.843
+            ],
+            [
+              -122,
+              37.843
+            ],
+            [
+              -122,
+              37.839
+            ],
+            [
+              -123,
+              37.839
+            ]
+          ]
+        ]
+      }
+    }
+  },
+  "layers": [
+    {
+      "id": "background",
+      "type": "background",
+      "paint": {
+        "background-color": "white"
+      }
+    },
+    {
+      "id": "raster",
+      "type": "raster",
+      "source": "contour",
+      "paint": {
+        "raster-fade-duration": 0
+      }
+    },
+    {
+      "id": "fill",
+      "type": "fill",
+      "source": "geojson",
+      "paint": {
+        "fill-antialias": false,
+        "fill-color": "green",
+        "fill-opacity": 0.2
+      }
+    },
+    {
+      "id": "raster-transparent",
+      "type": "raster",
+      "source": "contour",
+      "paint": {
+        "raster-fade-duration": 0,
+        "raster-opacity": 0.5,
+        "raster-hue-rotate": 90
+      }
+    }
+  ]
+}