diff --git a/pandoc-lua-engine/test/lua/module/pandoc-structure.lua b/pandoc-lua-engine/test/lua/module/pandoc-structure.lua
index 17209c7f8647..af19785c92a5 100644
--- a/pandoc-lua-engine/test/lua/module/pandoc-structure.lua
+++ b/pandoc-lua-engine/test/lua/module/pandoc-structure.lua
@@ -48,7 +48,7 @@ return {
       assert.are_equal('Div', hblks[1].t)
       assert.are_equal('Header', hblks[1].content[1].t)
       assert.are_equal('1',   hblks[1].content[1].attributes['number'])
-      assert.are_equal('1.1', hblks[1].content[3].attributes['number'])
+      assert.are_equal('1.0.1', hblks[1].content[3].attributes['number'])
     end)
   },
 
diff --git a/src/Text/Pandoc/Shared.hs b/src/Text/Pandoc/Shared.hs
index b18ec98c5c1c..1c26e143fd64 100644
--- a/src/Text/Pandoc/Shared.hs
+++ b/src/Text/Pandoc/Shared.hs
@@ -96,8 +96,9 @@ import Data.Char (isAlpha, isLower, isSpace, isUpper, toLower, isAlphaNum,
 import Data.List (find, foldl', groupBy, intercalate, intersperse,
                   union, sortOn)
 import qualified Data.Map as M
-import Data.Maybe (mapMaybe, fromMaybe)
-import Data.Monoid (Any (..))
+import Data.Maybe (mapMaybe)
+import Data.Monoid (Any (..) )
+import Data.Semigroup (Min (..))
 import Data.Sequence (ViewL (..), ViewR (..), viewl, viewr)
 import qualified Data.Set as Set
 import qualified Data.Text as T
@@ -507,26 +508,28 @@ textToIdentifier exts =
 -- element a Header).  If the 'numbering' parameter is True, Header
 -- numbers are added via the number attribute on the header.
 -- If the baseLevel parameter is Just n, Header levels are
--- adjusted to be gapless starting at level n.
+-- adjusted so that the lowest header level is n.
+-- (There may still be gaps in header level if the author leaves them.)
 makeSections :: Bool -> Maybe Int -> [Block] -> [Block]
 makeSections numbering mbBaseLevel bs =
-  S.evalState (go bs) (mbBaseLevel, [])
+  S.evalState (go bs) []
  where
-  go :: [Block] -> S.State (Maybe Int, [Int]) [Block]
+  getLevel (Header level _ _) = Min level
+  getLevel _ = Min 99
+  minLevel = getMin $ query getLevel bs
+  go :: [Block] -> S.State [Int] [Block]
   go (Header level (ident,classes,kvs) title':xs) = do
-    (mbLevel, lastnum) <- S.get
-    let level' = fromMaybe level mbLevel
+    lastnum <- S.get
+    let level' = maybe level (\n -> n + level - minLevel) mbBaseLevel
     let adjustNum lev numComponent
-          | lev < level' = numComponent
-          | lev == level' = numComponent + 1
+          | lev < level = numComponent
+          | lev == level = numComponent + 1
           | otherwise = 0
-    let newnum = zipWith adjustNum [(fromMaybe 1 mbBaseLevel)..level']
+    let newnum = zipWith adjustNum [minLevel..level]
                     (lastnum ++ repeat 0)
-    unless (null newnum) $ S.modify $ \(mbl, _) -> (mbl, newnum)
+    unless (null newnum) $ S.put newnum
     let (sectionContents, rest) = break (headerLtEq level) xs
-    S.modify $ \(_, ln) -> (fmap (+ 1) mbLevel, ln)
     sectionContents' <- go sectionContents
-    S.modify $ \(_, ln) -> (mbLevel, ln)
     rest' <- go rest
     let kvs' = -- don't touch number if already present
                case lookup "number" kvs of
diff --git a/src/Text/Pandoc/Writers/HTML.hs b/src/Text/Pandoc/Writers/HTML.hs
index 18e10f10237d..c7a4703ae404 100644
--- a/src/Text/Pandoc/Writers/HTML.hs
+++ b/src/Text/Pandoc/Writers/HTML.hs
@@ -721,17 +721,22 @@ adjustNumbers opts doc =
      then doc
      else walk go doc
   where
-   go (Div (ident,"section":classes,kvs) lst) =
-     Div (ident,"section":classes,map fixnum kvs) lst
+   go (Div (ident,"section":classes,kvs) lst@(Header level _ _ : _)) =
+     Div (ident,"section":classes,map (fixnum level) kvs) lst
    go (Header level (ident,classes,kvs) lst) =
-     Header level (ident,classes,map fixnum kvs) lst
+     Header level (ident,classes,map (fixnum level) kvs) lst
    go x = x
-   fixnum ("number",num) = ("number",
+   fixnum level ("number",num) = ("number",
                                showSecNum $ zipWith (+)
                                (writerNumberOffset opts ++ repeat 0)
-                               (map (fromMaybe 0 . safeRead) $
+                               (padTo level $
+                                map (fromMaybe 0 . safeRead) $
                                 T.split (=='.') num))
-   fixnum x = x
+   fixnum _ x = x
+   padTo n xs =
+     case n - length xs of
+       x | x > 0 -> replicate x 0 ++ xs
+         | otherwise -> xs
    showSecNum = T.intercalate "." . map tshow
 
 blockToHtmlInner :: PandocMonad m => WriterOptions -> Block -> StateT WriterState m Html
diff --git a/test/command/5071.md b/test/command/5071.md
new file mode 100644
index 000000000000..6bbeb85fcaa8
--- /dev/null
+++ b/test/command/5071.md
@@ -0,0 +1,68 @@
+```
+% pandoc -f markdown -t html --number-sections
+## First section
+
+### Subhead
+
+##### Subhead with gap
+
+## Second section
+^D
+<h2 data-number="1" id="first-section"><span
+class="header-section-number">1</span> First section</h2>
+<h3 data-number="1.1" id="subhead"><span
+class="header-section-number">1.1</span> Subhead</h3>
+<h5 data-number="1.1.0.1" id="subhead-with-gap"><span
+class="header-section-number">1.1.0.1</span> Subhead with gap</h5>
+<h2 data-number="2" id="second-section"><span
+class="header-section-number">2</span> Second section</h2>
+
+```
+
+```
+% pandoc -f markdown -t html --number-sections
+## First section
+
+### Subhead
+
+# Higher-level section
+
+## Sub
+^D
+<h2 data-number="0.1" id="first-section"><span
+class="header-section-number">0.1</span> First section</h2>
+<h3 data-number="0.1.1" id="subhead"><span
+class="header-section-number">0.1.1</span> Subhead</h3>
+<h1 data-number="1" id="higher-level-section"><span
+class="header-section-number">1</span> Higher-level section</h1>
+<h2 data-number="1.1" id="sub"><span
+class="header-section-number">1.1</span> Sub</h2>
+```
+
+For backwards compatibility, we want it to work the old way,
+giving numbers like 0.1, when `--number-offset` is used:
+```
+% pandoc -f markdown -t html --number-sections --number-offset=2,2,2
+## First section
+
+### Subhead
+^D
+<h2 data-number="2.3" id="first-section"><span
+class="header-section-number">2.3</span> First section</h2>
+<h3 data-number="2.3.3" id="subhead"><span
+class="header-section-number">2.3.3</span> Subhead</h3>
+
+```
+
+```
+% pandoc -f markdown -t html --number-sections --number-offset=0,2,2
+## First section
+
+### Subhead
+^D
+<h2 data-number="0.3" id="first-section"><span
+class="header-section-number">0.3</span> First section</h2>
+<h3 data-number="0.3.3" id="subhead"><span
+class="header-section-number">0.3.3</span> Subhead</h3>
+
+```