Improve loop unrolling documentation (#52099)

Minor cleanups to loop unrolling:
1. Add header comments with documentation
2. Make jitIterSmallOverflow/jitIterSmallUnderflow private static `Compiler`
class members, and not just "loose" functions.
3. Manually loop hoist ITER_LIMIT and UNROLL_LIMIT_SZ and dependent
asserts, plus `iterLimit` setting.

Also, make `optCloneLoops` and `optUnrollLoops` return a PhaseStatus,
so for early-out cases they can return MODIFIED_NOTHING.

src/coreclr/jit/compiler.h

@@ -6246,11 +6246,11 @@ class Compiler
     PhaseStatus optOptimizeLayout(); // Optimize the BasicBlock layout of the method
     PhaseStatus optFindLoops();      // Finds loops and records them in the loop table
 
-    void optCloneLoops();
+    PhaseStatus optCloneLoops();
     void optCloneLoop(unsigned loopInd, LoopCloneContext* context);
     void optEnsureUniqueHead(unsigned loopInd, BasicBlock::weight_t ambientWeight);
-    void optUnrollLoops(); // Unrolls loops (needs to have cost info)
-    void optRemoveRedundantZeroInits();
+    PhaseStatus optUnrollLoops(); // Unrolls loops (needs to have cost info)
+    void        optRemoveRedundantZeroInits();
 
 protected:
     // This enumeration describes what is killed by a call.
@@ -6580,6 +6580,10 @@ class Compiler
 
     void optInvertWhileLoop(BasicBlock* block);
 
+private:
+    static bool optIterSmallOverflow(int iterAtExit, var_types incrType);
+    static bool optIterSmallUnderflow(int iterAtExit, var_types decrType);
+
     bool optComputeLoopRep(int        constInit,
                            int        constLimit,
                            int        iterInc,
@@ -6590,7 +6594,6 @@ class Compiler
                            bool       dupCond,
                            unsigned*  iterCount);
 
-private:
     static fgWalkPreFn optIsVarAssgCB;
 
 protected: