first commit

.gitignore

@@ -0,0 +1,3 @@
+model/
+log*/
+__pycache__/

README.md

@@ -0,0 +1,111 @@
+# DeepInverseRendering
+
+Source code of our paper: 
+
+Deep Inverse Rendering for High-resolution SVBRDF Estimation from an Arbitrary Number of Images
+
+> Duan Gao, Xiao Li, Yue Dong, Pieter Peers, Kun Xu, Xin Tong.
+>
+> ACM Transactions on Graphics (Proceedings of SIGGRAPH 2019)*
+
+
+![image](mvsvbrdf.png)
+
+**More information (including our paper, supplementary, video and slides) can be found at [My Personal Page](https://gao-duan.github.io/).**
+
+If you have any questions about our paper, please feel free to contact me (gao-d17@mails.tsinghua.edu.cn)
+
+
+
+## Pretrained models
+
+Our pretrained SVBRDF auto-encoder can be downloaded from [here](https://drive.google.com/open?id=17WrAZIkyejwRm5aKBtTvNHYe_3c6S9aE).
+
+
+
+## Dependencies 
+
+- Python (with opencv-python, numpy; test on Python 3.6)
+
+- Tensorflow-gpu (test on tensorflow>1.10)
+
+
+
+## Usage
+
+- Eval mode (captured images as input)
+
+```bash
+   python3 main.py 
+      --N $input_numbers            # number of input images
+      --checkpoint $checkpoint      # pretrained model of auto-encoder
+      --dataDir $images_data_dir    # folder contains a set of input images
+      --logDir $log_dir             # output folder
+      --initDir  $init_dir          # folder contains initial SVBRDF maps or initial code 
+      --network $network            # network architecture (default: network_ae_fixBN)
+
+      --init_method svbrdf          
+      --input_type image 
+      --wlv_type $wlv_dir 
+```
+
+- Test mode (SVBRDF map as input)
+```bash
+python3 main.py 
+   --N $input_numbers 
+   --checkpoint $checkpoint 
+   --dataDir $svbrdf_data_dir 
+   --logDir $log_dir 
+   --initDir  $init_dir 
+   --network $network  
+
+   --init_method svbrdf 
+   --input_type svbrdf 
+   --wlv_type random 
+```
+
+- Other command arguments:
+
+```
+-- input_type: ['image', 'svbrdf']
+   
+   'image':  a set of images, used in evaluate.
+   
+   'svbrdf':  a set of SVBRDFs, used in testing.
+   
+   
+-- init_method: ['svbrdf', 'code', 'rand']
+
+ 'svbrdf': the estimated SVBRDF (using our encoder to embedding it into our latent     space)
+ 
+ 'code':  the latent code (numpy array)
+ 
+ 'rand': random initialization the latent code
+
+
+-- wlv_type: ['random', 'load']
+   
+   'random': random generate camera position and light position
+   
+   'load': load camera position and light position from file
+```
+
+- Example data can be found in `example_data` folder.
+
+## Citation
+
+If you use our code or pretrained models, please cite as following:
+
+```
+@article{gao2019deep,
+  title={Deep inverse rendering for high-resolution SVBRDF estimation from an arbitrary number of images},
+  author={GAO, DUAN and Li, Xiao and Dong, Yue and Peers, Pieter and Xu, Kun and Tong, Xin},
+  journal={ACM Transactions on Graphics (TOG)},
+  volume={38},
+  number={4},
+  pages={134},
+  year={2019},
+  month={July},
+  publisher={ACM}
+}
+```

example_data/example_images/files.txt

@@ -0,0 +1 @@
+0000066;leather_tilesXPolishedMarbleFloor_01;defaultX4

example_data/example_svbrdf/files.txt

@@ -0,0 +1,2 @@
+0000066;leather_tilesXPolishedMarbleFloor_01;defaultX4
+0000080;PolishedMarbleFloor_01Xbrick_uneven_stones;1Xdefault

example_data/example_wlv/0000066;leather_tilesXPolishedMarbleFloor_01;defaultX4.txt

@@ -0,0 +1,21 @@
+predefined N: 20
+0,0,2.146	0,0,2.146
+-0.8984032091774394,1.1283482369324698,2.146	-0.8984032091774394,1.1283482369324698,2.146
+0.40302757472370954,-1.1638627131150379,2.146	0.40302757472370954,-1.1638627131150379,2.146
+-0.9869832659564078,0.5701036222373868,2.146	-0.9869832659564078,0.5701036222373868,2.146
+-1.1198079696256764,0.43548663962711687,2.146	-1.1198079696256764,0.43548663962711687,2.146
+-0.4140864846438477,-0.4781888059004863,2.146	-0.4140864846438477,-0.4781888059004863,2.146
+0.3025837023116329,-1.0337924460156456,2.146	0.3025837023116329,-1.0337924460156456,2.146
+-0.18131583911173088,-1.1097281992533012,2.146	-0.18131583911173088,-1.1097281992533012,2.146
+-1.1592029307608742,-0.8132741606510094,2.146	-1.1592029307608742,-0.8132741606510094,2.146
+0.4283604126646985,-0.641717911436314,2.146	0.4283604126646985,-0.641717911436314,2.146
+0.38519378155258344,-0.4336049536059965,2.146	0.38519378155258344,-0.4336049536059965,2.146
+0.304720670854038,0.1288269438954195,2.146	0.304720670854038,0.1288269438954195,2.146
+-1.0200770193437456,0.5927958220976814,2.146	-1.0200770193437456,0.5927958220976814,2.146
+0.6678770146697104,-0.5715322466321664,2.146	0.6678770146697104,-0.5715322466321664,2.146
+-0.2733197235315623,-0.41693967566959855,2.146	-0.2733197235315623,-0.41693967566959855,2.146
+0.5242429064404166,-0.9753501673990478,2.146	0.5242429064404166,-0.9753501673990478,2.146
+-1.1542805014601494,-1.0653621891415144,2.146	-1.1542805014601494,-1.0653621891415144,2.146
+1.0261447040237999,-0.2707309808189786,2.146	1.0261447040237999,-0.2707309808189786,2.146
+-0.5500861102083011,0.2413367163248732,2.146	-0.5500861102083011,0.2413367163248732,2.146
+-0.6581809514318763,-0.13377781200108085,2.146	-0.6581809514318763,-0.13377781200108085,2.146

example_data/example_wlv/0000080;PolishedMarbleFloor_01Xbrick_uneven_stones;1Xdefault.txt

@@ -0,0 +1,21 @@
+predefined N: 20
+0,0,2.146	0,0,2.146
+-0.8984032091774394,1.1283482369324698,2.146	-0.8984032091774394,1.1283482369324698,2.146
+0.40302757472370954,-1.1638627131150379,2.146	0.40302757472370954,-1.1638627131150379,2.146
+-0.9869832659564078,0.5701036222373868,2.146	-0.9869832659564078,0.5701036222373868,2.146
+-1.1198079696256764,0.43548663962711687,2.146	-1.1198079696256764,0.43548663962711687,2.146
+-0.4140864846438477,-0.4781888059004863,2.146	-0.4140864846438477,-0.4781888059004863,2.146
+0.3025837023116329,-1.0337924460156456,2.146	0.3025837023116329,-1.0337924460156456,2.146
+-0.18131583911173088,-1.1097281992533012,2.146	-0.18131583911173088,-1.1097281992533012,2.146
+-1.1592029307608742,-0.8132741606510094,2.146	-1.1592029307608742,-0.8132741606510094,2.146
+0.4283604126646985,-0.641717911436314,2.146	0.4283604126646985,-0.641717911436314,2.146
+0.38519378155258344,-0.4336049536059965,2.146	0.38519378155258344,-0.4336049536059965,2.146
+0.304720670854038,0.1288269438954195,2.146	0.304720670854038,0.1288269438954195,2.146
+-1.0200770193437456,0.5927958220976814,2.146	-1.0200770193437456,0.5927958220976814,2.146
+0.6678770146697104,-0.5715322466321664,2.146	0.6678770146697104,-0.5715322466321664,2.146
+-0.2733197235315623,-0.41693967566959855,2.146	-0.2733197235315623,-0.41693967566959855,2.146
+0.5242429064404166,-0.9753501673990478,2.146	0.5242429064404166,-0.9753501673990478,2.146
+-1.1542805014601494,-1.0653621891415144,2.146	-1.1542805014601494,-1.0653621891415144,2.146
+1.0261447040237999,-0.2707309808189786,2.146	1.0261447040237999,-0.2707309808189786,2.146
+-0.5500861102083011,0.2413367163248732,2.146	-0.5500861102083011,0.2413367163248732,2.146
+-0.6581809514318763,-0.13377781200108085,2.146	-0.6581809514318763,-0.13377781200108085,2.146

optimization/data_util.py

@@ -0,0 +1,93 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import numpy as np
+import cv2
+import os
+import glob
+from ops import *
+import tensorflow as tf
+
+"""
+load svbrdf
+file extension:
+    .npyimage.npy: float32 numpy array
+    .png: uint8 image.
+"""
+def load_svbrdf(name, scale_size, data_format):
+    if data_format == 'npy':
+        tmp_path, tmp_name = os.path.split(name)
+        tmp_name = os.path.splitext(tmp_name)[0]
+        new_name = os.path.join(tmp_path, '%s.npyimage.npy'%tmp_name)
+        # if not exist, use .png instead
+        if not os.path.exists(new_name):
+            print("[Warning] npy image %s not exist, load png instead. %s" % (new_name,name))
+            img = cv2.imread(name)
+            if img is None:
+                print(name)
+            img = img.astype(np.float32) / 255.0
+            img = img[:,:,::-1]
+        else:
+            img = np.load(new_name)
+    else:
+        img = cv2.imread(name)
+        img = img.astype(np.float32) / 255.0
+        img = img[:,:,::-1]
+
+
+    h,w, _ = img.shape
+    nb_imgs = w // h
+    image_width = w // nb_imgs
+
+    images = []
+    for i in range(4):
+        tmp = img[:, i * image_width: (i+1) * image_width, :]
+        if i == 1:
+            tmp = tmp ** 2.2        # diffuse: srgb => linear
+        images.append(tmp)  
+
+    new_images = []
+    for img in images:
+        img = img * 2.0 - 1.0
+        if h != scale_size:
+            if h > scale_size:
+                print("[info] AREA resize image from %d to %d" % (h, scale_size))
+                img = cv2.resize(img, (scale_size, scale_size), interpolation=cv2.INTER_AREA)
+            else:
+                print("[info] LINEAR resize image from %d to %d" % (h, scale_size))
+                img = cv2.resize(img, (scale_size, scale_size))
+        new_images.append(img)
+    images = new_images
+
+    # 4 x [size, size, 3] => [size, size, 12]
+    res =  np.concatenate(images, axis = -1)
+
+    # [1,size,size,12]
+    res = res[np.newaxis,...]
+    return res
+
+
+def load_input_img(name, data_format):
+    if data_format == 'npy':
+        tmp_path, tmp_name = os.path.split(name)
+        tmp_name = os.path.splitext(tmp_name)[0]
+        new_name = os.path.join(tmp_path, '%s.npyimage.npy'%tmp_name)
+        if os.path.exists(new_name):
+            img = np.load(new_name)
+            img = img[...,::-1]
+        else:
+            print("[Warning] npy image %s not exist, load png instead. %s" % (new_name,name))
+            img = cv2.imread(name)
+            img = img.astype(np.float32) / 255.0
+            img = img[...,::-1]
+    else:
+        img = cv2.imread(name)
+        img = img.astype(np.float32) / 255.0
+        img = img[...,::-1]
+    return img
+
+def load_files(path):
+    files =  glob.glob(os.path.join(path, "*.png"))
+    files.extend(glob.glob(os.path.join(path, "*.jpg")))
+    return files
+

optimization/loss_util.py

@@ -0,0 +1,15 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import tensorflow as tf
+
+
+def L1Loss(inputs, targets, weight = 1.0):
+    diff = tf.abs(inputs - targets)
+    return tf.reduce_mean(diff) * weight
+
+def L1LogLoss(inputs, targets, weight = 1.0):
+    return tf.reduce_mean(tf.abs(tf.log(inputs + 0.01) - tf.log(targets + 0.01))) * weight
+
+def L2Loss(inputs, targets,  weight = 1.0):
+    return tf.reduce_mean(tf.squared_difference(inputs, targets)) * weight

optimization/main.py

@@ -0,0 +1,101 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import tensorflow as tf
+import argparse
+import importlib
+from optimizer import Optimizer
+from data_util import load_files
+import os
+import numpy as np
+import random
+
+parser = argparse.ArgumentParser()
+
+parser.add_argument("--dataDir", "-d", required=True)
+parser.add_argument("--logDir", "-l", required=True)
+parser.add_argument("--initDir", type=str, required=True)
+parser.add_argument("--init_method", type=str, required=True, choices=['rand','code','svbrdf'])
+parser.add_argument("--network", type=str, required=True)
+parser.add_argument("--checkpoint", type=str, required=True)
+parser.add_argument("--N", type=int,required=True)
+
+parser.add_argument("--wlvDir", type=str, default=None)
+parser.add_argument("--exposure_scale", type=float, default=1.0)
+parser.add_argument("--input_type", type=str, default='svbrdf', choices=['svbrdf','image'])
+parser.add_argument("--wlv_type", type=str, default='random', choices=['load','random'])
+parser.add_argument("--LDR", dest = "LDR", action = "store_true")
+parser.add_argument("--not-LDR", dest = "LDR", action = "store_false")
+parser.set_defaults(LDR=False)
+parser.add_argument("--seed", type=int, default=20181120)
+parser.add_argument("--refine_init", type=str, default='npy', choices=['npy','image'])
+
+
+parser.add_argument("--scale_size", type = int,default=256)
+parser.add_argument("--max_steps", type = int, default = 4000)
+parser.add_argument("--refine_max_steps", type = int, default = 200)
+parser.add_argument("--lr", type=float, default=1e-3)
+parser.add_argument("--beta1", type=float, default=0.5)
+parser.add_argument("--progress_freq", type = int, default = 500)
+parser.add_argument("--save_freq", type = int, default = 500)
+parser.add_argument("--refine_progress_freq", type = int, default = 50)
+parser.add_argument("--refine_save_freq", type = int, default = 50)
+parser.add_argument("--initial_mu", type=float, default=0.0)
+parser.add_argument("--initial_std", type=float, default=1.0)
+parser.add_argument("--data_format", type=str, default='npy')
+parser.add_argument("--data_loss_type", type=str, choices=['l1' ,'l1log','test'], default='l1log')
+
+args,unknown = parser.parse_known_args()
+
+def set_seed():
+    if args.seed == -1:
+        args.seed = random.randint(0, 2 ** 31 - 1)
+    tf.set_random_seed(args.seed)
+    np.random.seed(args.seed)
+    random.seed(args.seed)
+
+def main():
+    set_seed()
+    print(args)
+    print(unknown)
+
+    network = importlib.import_module(args.network)
+
+    optimizer = Optimizer(network.code_shape,
+                    network.encoder_var_names,
+                    network.decoder_var_names,
+                    network.encoder,
+                    network.decoder, 
+                    args)
+
+    BaseNames = []
+    with open(os.path.join(args.dataDir, 'files.txt'),'r') as f:
+        for line in f:
+            BaseNames.append(line.strip())
+
+    for f in BaseNames:
+        set_seed()
+        print('[INFO]' + f)
+
+        output_folder = os.path.join(args.logDir, f)
+        refine_output_folder = os.path.join(args.logDir, f + '_refine')
+
+        if not os.path.exists(output_folder):
+            os.mkdir(output_folder)
+        if not os.path.exists(refine_output_folder):
+            os.mkdir(refine_output_folder)
+
+        if os.path.exists(os.path.join(output_folder, 'log.txt')):
+            print('Already done ae opt. ignore: %s' % output_folder)
+            if os.path.exists(os.path.join(refine_output_folder, 'log.txt')):
+                print('Already done refine opt. ignore: %s' % output_folder)
+            else:
+                optimizer.refine_opt(f, refine_output_folder, output_folder)
+        else:
+            optimizer.ae_opt(f, output_folder)
+            optimizer.refine_opt(f, refine_output_folder, output_folder)
+
+if __name__ == "__main__":
+    if not os.path.exists(args.logDir):
+        os.makedirs(args.logDir)
+    main()