Merge branch 'master' into smola/appsec-rules-1.11.0

dd-java-agent/agent-iast/src/main/java/com/datadog/iast/propagation/FastCodecModule.java

@@ -1,5 +1,7 @@
 package com.datadog.iast.propagation;
 
+import static datadog.trace.api.iast.VulnerabilityMarks.NOT_MARKED;
+
 import datadog.trace.api.iast.propagation.CodecModule;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
@@ -14,8 +16,13 @@ public void onUrlDecode(
 
   @Override
   public void onStringFromBytes(
-      @Nonnull final byte[] value, @Nullable final String charset, @Nonnull final String result) {
-    taintIfTainted(result, value);
+      @Nonnull final byte[] value,
+      int offset,
+      int length,
+      @Nullable final String charset,
+      @Nonnull final String result) {
+    // create a new range shifted to the result string coordinates
+    taintIfTainted(result, value, offset, length, false, NOT_MARKED);
   }
 
   @Override

dd-java-agent/agent-iast/src/main/java/com/datadog/iast/propagation/PropagationModuleImpl.java

@@ -12,6 +12,7 @@
 import com.datadog.iast.taint.TaintedObjects;
 import com.datadog.iast.taint.Tainteds;
 import com.datadog.iast.util.ObjectVisitor;
+import com.datadog.iast.util.Ranged;
 import datadog.trace.api.Config;
 import datadog.trace.api.iast.IastContext;
 import datadog.trace.api.iast.Taintable;
@@ -57,6 +58,27 @@ public void taint(
     taint(ctx, target, origin, null);
   }
 
+  @Override
+  public void taint(
+      @Nullable final Object target, final byte origin, final int start, final int length) {
+    taint(LazyContext.build(), target, origin, start, length);
+  }
+
+  @Override
+  public void taint(
+      @Nullable final IastContext ctx,
+      @Nullable final Object target,
+      final byte origin,
+      final int start,
+      final int length) {
+    if (!canBeTainted(target) || length == 0) {
+      return;
+    }
+    final Range range =
+        new Range(start, length, newSource(target, origin, null, target), NOT_MARKED);
+    internalTaint(ctx, target, new Range[] {range}, NOT_MARKED);
+  }
+
   @Override
   public void taint(
       @Nullable final IastContext ctx,
@@ -118,6 +140,45 @@ public void taintIfTainted(
     }
   }
 
+  @Override
+  public void taintIfTainted(
+      @Nullable final Object target,
+      @Nullable final Object input,
+      final int start,
+      final int length,
+      boolean keepRanges,
+      int mark) {
+    taintIfTainted(LazyContext.build(), target, input, start, length, keepRanges, mark);
+  }
+
+  @Override
+  public void taintIfTainted(
+      @Nullable final IastContext ctx,
+      @Nullable final Object target,
+      @Nullable final Object input,
+      final int start,
+      final int length,
+      boolean keepRanges,
+      int mark) {
+    if (!canBeTainted(target) || !canBeTainted(input) || length == 0) {
+      return;
+    }
+    final Range[] ranges = getRanges(ctx, input);
+    if (ranges == null || ranges.length == 0) {
+      return;
+    }
+    final Range[] intersection = Ranges.intersection(Ranged.build(start, length), ranges);
+    if (intersection == null || intersection.length == 0) {
+      return;
+    }
+    if (keepRanges) {
+      internalTaint(ctx, target, intersection, mark);
+    } else {
+      final Range range = highestPriorityRange(intersection);
+      internalTaint(ctx, target, range.getSource(), mark);
+    }
+  }
+
   @Override
   public void taintIfTainted(
       @Nullable final Object target, @Nullable final Object input, final byte origin) {
@@ -436,15 +497,26 @@ private static void internalTaint(
       return;
     }
     if (value instanceof Taintable) {
-      ((Taintable) value).$$DD$setSource(ranges[0].getSource());
+      final Taintable taintable = (Taintable) value;
+      if (!taintable.$DD$isTainted()) {
+        taintable.$$DD$setSource(ranges[0].getSource());
+      }
     } else {
       final TaintedObjects to = getTaintedObjects(ctx);
       if (to != null) {
         if (value instanceof CharSequence) {
           ranges = attachSourceValue(ranges, (CharSequence) value);
         }
         ranges = markRanges(ranges, mark);
-        to.taint(value, ranges);
+        final TaintedObject tainted = to.get(value);
+        if (tainted == null) {
+          // taint new value
+          to.taint(value, ranges);
+        } else {
+          // append ranges
+          final Range[] newRanges = Ranges.mergeRangesSorted(tainted.getRanges(), ranges);
+          tainted.setRanges(newRanges);
+        }
       }
     }
   }

dd-java-agent/agent-iast/src/main/java/com/datadog/iast/taint/Ranges.java

@@ -6,6 +6,8 @@
 import com.datadog.iast.model.Range;
 import com.datadog.iast.model.Source;
 import com.datadog.iast.util.HttpHeader;
+import com.datadog.iast.util.RangeBuilder;
+import com.datadog.iast.util.Ranged;
 import datadog.trace.api.iast.SourceTypes;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
@@ -35,6 +37,30 @@ public static Range[] forObject(final @Nonnull Source source, final int mark) {
     return new Range[] {new Range(0, Integer.MAX_VALUE, source, mark)};
   }
 
+  @Nullable
+  public static Range[] intersection(
+      final @Nonnull Ranged targetRange, @Nonnull final Range[] ranges) {
+    final Range last = ranges[ranges.length - 1];
+    final int lastIndex = last.getStart() + last.getLength();
+
+    final RangeBuilder targetRanges = new RangeBuilder(ranges.length);
+    for (final Range range : ranges) {
+      if (range.getStart() >= lastIndex) {
+        break;
+      }
+      final Ranged intersection = targetRange.intersection(range);
+      if (intersection != null) {
+        targetRanges.add(
+            new Range(
+                intersection.getStart(),
+                intersection.getLength(),
+                range.getSource(),
+                range.getMarks()));
+      }
+    }
+    return targetRanges.isEmpty() ? null : targetRanges.toArray();
+  }
+
   @Nullable
   public static Range findUnbound(@Nonnull final Range[] ranges) {
     if (ranges.length != 1) {
@@ -217,6 +243,36 @@ public static Range[] newArray(final long size) {
     return new Range[size > MAX_RANGE_COUNT ? MAX_RANGE_COUNT : (int) size];
   }
 
+  /** Merge the new ranges maintaining order (it assumes that both arrays are already sorted) */
+  public static Range[] mergeRangesSorted(final Range[] leftRanges, final Range[] rightRanges) {
+    if (leftRanges.length == 0) {
+      return rightRanges;
+    }
+    if (rightRanges.length == 0) {
+      return leftRanges;
+    }
+    int rightIndex = 0;
+    Range rightRange = rightRanges[0];
+    final Range[] result = new Range[rightRanges.length + leftRanges.length];
+    for (int leftIndex = 0; leftIndex < leftRanges.length; leftIndex++) {
+      final Range leftRange = leftRanges[leftIndex];
+      if (rightRange == null) {
+        result[leftIndex + rightRanges.length] = leftRange;
+      } else {
+        if (rightRange.getStart() < leftRange.getStart()) {
+          result[leftIndex + rightIndex] = rightRange;
+          rightIndex++;
+          rightRange = rightIndex >= rightRanges.length ? null : rightRanges[rightIndex];
+        }
+        result[leftIndex + rightIndex] = leftRange;
+      }
+    }
+    for (; rightIndex < rightRanges.length; rightIndex++) {
+      result[leftRanges.length + rightIndex] = rightRanges[rightIndex];
+    }
+    return result;
+  }
+
   @Nullable
   public static Range[] getNotMarkedRanges(@Nullable final Range[] ranges, final int mark) {
     if (ranges == null) {

dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/propagation/BaseCodecModuleTest.groovy

@@ -11,7 +11,7 @@ import static com.datadog.iast.taint.TaintUtils.addFromTaintFormat
 @CompileDynamic
 abstract class BaseCodecModuleTest extends IastModuleImplTestBase {
 
-  private CodecModule module
+  protected CodecModule module
 
   def setup() {
     module = buildModule()
@@ -38,8 +38,8 @@ abstract class BaseCodecModuleTest extends IastModuleImplTestBase {
     'onUrlDecode'       | ['test', 'utf-8', '']
     'onStringGetBytes'  | ['test', 'utf-8', null]
     'onStringGetBytes'  | ['test', 'utf-8', [] as byte[]]
-    'onStringFromBytes' | ['test'.bytes, 'utf-8', null]
-    'onStringFromBytes' | ['test'.bytes, 'utf-8', '']
+    'onStringFromBytes' | ['test'.bytes, 0, 2, 'utf-8', null]
+    'onStringFromBytes' | ['test'.bytes, 0, 2, 'utf-8', '']
     'onBase64Encode'    | ['test'.bytes, null]
     'onBase64Encode'    | ['test'.bytes, [] as byte[]]
     'onBase64Decode'    | ['test'.bytes, null]
@@ -57,7 +57,7 @@ abstract class BaseCodecModuleTest extends IastModuleImplTestBase {
     method              | args
     'onUrlDecode'       | ['test', 'utf-8', 'decoded']
     'onStringGetBytes'  | ['test', 'utf-8', 'test'.getBytes('utf-8')]
-    'onStringFromBytes' | ['test'.getBytes('utf-8'), 'utf-8', 'test']
+    'onStringFromBytes' | ['test'.getBytes('utf-8'), 0, 2, 'utf-8', 'test']
     'onBase64Encode'    | ['test'.bytes, 'dGVzdA=='.bytes]
     'onBase64Decode'    | ['dGVzdA=='.bytes, 'test'.bytes]
   }
@@ -138,15 +138,15 @@ abstract class BaseCodecModuleTest extends IastModuleImplTestBase {
     final result = charset == null ? new String(bytes) : new String(bytes, (String) charset)
 
     when:
-    module.onStringFromBytes(bytes, charset, result)
+    module.onStringFromBytes(bytes, 0, bytes.length, charset, result)
 
     then:
     final to = taintedObjects.get(result)
     if (isTainted) {
       assert to != null
       assert to.get() == result
       final sourceTainted = taintedObjects.get(parsed)
-      assertOnStringFromBytes(bytes, charset, sourceTainted, to)
+      assertOnStringFromBytes(bytes, 0, bytes.length, charset, sourceTainted, to)
     } else {
       assert to == null
     }
@@ -239,7 +239,7 @@ abstract class BaseCodecModuleTest extends IastModuleImplTestBase {
 
   protected abstract void assertOnUrlDecode(final String value, final String encoding, final TaintedObject source, final TaintedObject target)
 
-  protected abstract void assertOnStringFromBytes(final byte[] value, final String encoding, final TaintedObject source, final TaintedObject target)
+  protected abstract void assertOnStringFromBytes(final byte[] value, final int offset, final int length, final String encoding, final TaintedObject source, final TaintedObject target)
 
   protected abstract void assertOnStringGetBytes(final String value, final String encoding, final TaintedObject source, final TaintedObject target)
 

dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/propagation/FastCodecModuleTest.groovy

@@ -1,8 +1,14 @@
 package com.datadog.iast.propagation
 
+import com.datadog.iast.model.Source
 import com.datadog.iast.taint.Ranges
 import com.datadog.iast.taint.TaintedObject
+import com.datadog.iast.taint.TaintedObjects
+import datadog.trace.api.iast.VulnerabilityMarks
 import datadog.trace.api.iast.propagation.CodecModule
+import com.datadog.iast.model.Range
+
+import java.nio.charset.StandardCharsets
 
 class FastCodecModuleTest extends BaseCodecModuleTest {
 
@@ -24,7 +30,7 @@ class FastCodecModuleTest extends BaseCodecModuleTest {
   }
 
   @Override
-  protected void assertOnStringFromBytes(final byte[] value, final String charset, final TaintedObject source, final TaintedObject target) {
+  protected void assertOnStringFromBytes(final byte[] value, final int offset, final int length, final String charset, final TaintedObject source, final TaintedObject target) {
     final result = target.get() as String
     assert target.ranges.size() == 1
 
@@ -67,4 +73,43 @@ class FastCodecModuleTest extends BaseCodecModuleTest {
     assert range.length == Integer.MAX_VALUE // unbound for non char sequences
     assert range.source == sourceRange.source
   }
+
+  void 'test on string from bytes with multiple ranges'() {
+    given:
+    final charset = StandardCharsets.UTF_8
+    final string = "Hello World!"
+    final bytes = string.getBytes(charset) // 1 byte pe char
+    final TaintedObjects to = ctx.taintedObjects
+    final ranges = [
+      new Range(0, 5, new Source((byte) 0, 'name1', 'Hello'), VulnerabilityMarks.NOT_MARKED),
+      new Range(6, 6, new Source((byte) 1, 'name2', 'World!'), VulnerabilityMarks.NOT_MARKED)
+    ]
+    to.taint(bytes, ranges as Range[])
+
+    when:
+    final hello = string.substring(0, 5)
+    module.onStringFromBytes(bytes, 0, 5, charset.name(), hello)
+
+    then:
+    final helloTainted = to.get(hello)
+    helloTainted.ranges.length == 1
+    helloTainted.ranges.first().with {
+      assert it.source.origin == (byte) 0
+      assert it.source.name == 'name1'
+      assert it.source.value == 'Hello'
+    }
+
+    when:
+    final world = string.substring(6, 12)
+    module.onStringFromBytes(bytes, 6, 6, charset.name(), world)
+
+    then:
+    final worldTainted = to.get(world)
+    worldTainted.ranges.length == 1
+    worldTainted.ranges.first().with {
+      assert it.source.origin == (byte) 1
+      assert it.source.name == 'name2'
+      assert it.source.value == 'World!'
+    }
+  }
 }