\documentclass[10pt]{article}

\usepackage{fullpage}
\usepackage{setspace}
\usepackage{parskip}
\usepackage{titlesec}
\usepackage[section]{placeins}
\usepackage{xcolor}
\usepackage{breakcites}
\usepackage{lineno}
\usepackage{hyphenat}





\PassOptionsToPackage{hyphens}{url}
\usepackage[colorlinks = true,
            linkcolor = blue,
            urlcolor  = blue,
            citecolor = blue,
            anchorcolor = blue]{hyperref}
\usepackage{etoolbox}
\makeatletter
\patchcmd\@combinedblfloats{\box\@outputbox}{\unvbox\@outputbox}{}{%
  \errmessage{\noexpand\@combinedblfloats could not be patched}%
}%
\makeatother


\usepackage{natbib}




\renewenvironment{abstract}
  {{\bfseries\noindent{\abstractname}\par\nobreak}\footnotesize}
  {\bigskip}

\titlespacing{\section}{0pt}{*3}{*1}
\titlespacing{\subsection}{0pt}{*2}{*0.5}
\titlespacing{\subsubsection}{0pt}{*1.5}{0pt}


\usepackage{authblk}


\usepackage{graphicx}
\usepackage[space]{grffile}
\usepackage{latexsym}
\usepackage{textcomp}
\usepackage{longtable}
\usepackage{tabulary}
\usepackage{booktabs,array,multirow}
\usepackage{amsfonts,amsmath,amssymb}
\providecommand\citet{\cite}
\providecommand\citep{\cite}
\providecommand\citealt{\cite}
% You can conditionalize code for latexml or normal latex using this.
\newif\iflatexml\latexmlfalse
\providecommand{\tightlist}{\setlength{\itemsep}{0pt}\setlength{\parskip}{0pt}}%

\AtBeginDocument{\DeclareGraphicsExtensions{.pdf,.PDF,.eps,.EPS,.png,.PNG,.tif,.TIF,.jpg,.JPG,.jpeg,.JPEG}}

\usepackage[utf8]{inputenc}
\usepackage[ngerman,greek,english]{babel}



\usepackage{float}








\begin{document}

\title{Multiscale fusion of digital rock images based on deep generative
adversarial networks}



\author[1]{Mingliang Liu}%
\author[1]{Tapan Mukerji}%
\affil[1]{Stanford University}%


\vspace{-1em}



  
  \date{May 22, 2024}


\begingroup
\let\center\flushleft
\let\endcenter\endflushleft
\maketitle
\endgroup





\selectlanguage{english}
\begin{abstract}
Computation of petrophysical properties on digital rock images is
becoming important in geoscience. However, it is usually complicated for
natural heterogeneous porous media due to the presence of multiscale
pore structures. To capture the heterogeneity of rocks, we develop a
method based on deep generative adversarial networks to assimilate
multiscale imaging data for the generation of synthetic high-resolution
digital rocks having a large field of view. The reconstructed images not
only honor the geometric structures of 3-D micro-CT images but also
recover fine details existing at the scale of 2-D scanning electron
microscopy images. Furthermore, the consistency between the real and
synthetically generated images in terms of porosity, specific surface
area, two-point correlation and effective permeability reveals the
validity of our proposed method. It provides an effective way to fuse
multiscale digital rock images for better characterization of
heterogeneous porous media and better prediction of pore-scale flow and
petrophysical properties.%
\end{abstract}%



\sloppy


\textbf{Multiscale fusion of digital rock images based on deep
generative adversarial networks}

\textbf{Mingliang Liu\textsuperscript{1}} and \textbf{Tapan
Mukerji\textsuperscript{1}}

\textsuperscript{1} Department of Energy Resources Engineering, Stanford
University, USA.

Corresponding author: Mingliang Liu (mliu9@stanford.edu)

\textbf{Key Points:}

\begin{itemize}
\tightlist
\item
  Style-based generative adversarial network (GAN) is effective for
  augmenting sparse high-resolution SEM images.
\item
  Cycle-consistent GAN is successfully applied to integrate unpaired
  multi-resolution digital rock images.
\item
  The method shows promise in the reconstruction of high-resolution
  digital rocks that accurately capture micro-structures at
  multi-scales.
\end{itemize}

\textbf{ABSTRACT}

Computation of petrophysical properties on digital rock images is
becoming important in geoscience. However, it is usually complicated for
natural heterogeneous porous media due to the presence of multiscale
pore structures. To capture the heterogeneity of rocks, we develop a
method based on deep generative adversarial networks to assimilate
multiscale imaging data for the generation of synthetic high-resolution
digital rocks having a large field of view. The reconstructed images not
only honor the geometric structures of 3-D micro-CT images but also
recover fine details existing at the scale of 2-D scanning electron
microscopy images. Furthermore, the consistency between the real and
synthetically generated images in terms of porosity, specific surface
area, two-point correlation and effective permeability reveals the
validity of our proposed method. It provides an effective way to fuse
multiscale digital rock images for better characterization of
heterogeneous porous media and better prediction of pore-scale flow and
petrophysical properties.

\textbf{Plain Language Summary}

Digital rock physics is an effective approach to characterize pore
microstructures and predict effective physical properties of porous
medium. However, there is an inherent trade-off between imaging
resolution and field of view (FoV) due to the limitations of different
imaging techniques: high-resolution data can resolve pore structures up
to nano scale but the FoV is not large enough to capture a
representative volume element; low-resolution data have larger FoV but
cannot capture the fine features. To overcome the trade-off, it is
necessary to develop a workflow of multiscale data fusion for the
reconstruction of digital rocks with both high-resolution and large FoV.
The major challenges are (1) that the high-resolution images are usually
limited in number and (2) that the imaging data are typically acquired
at different locations of the rock sample, which means that the training
samples are unpaired. To address such challenges, we use a style-based
generative adversarial network (GAN) to augment the limited
high-resolution data and then use a cycle-consistent GAN to integrate
the unpaired digital rock data from multiple sources. The proposed
method performs well at reconstructing high-resolution rock models that
allows more accurate fluid flow simulation at pore-scale and prediction
of effective properties.

\textbf{Introduction}

Having a deep understanding of pore-scale processes in porous media is
of critical importance for various subsurface applications, such as
water resources, oil and gas recovery and carbon dioxide sequestration.
The conventional method that takes the physical experiments on core
plugs are typically time-consuming and limited by the lab environment.
As an alternative approach, digital rock physics (DRP) performs
numerical simulations of pore-scale processes of interest directly on
digital scans of porous rocks (Keehm et al., 2001 and 2004; Andr\selectlanguage{ngerman}ä et
al., 2013a and 2013b; Blunt et al., 2013; Saxena et al., 2017). It
provides a non-destructive means of repeatedly carrying out numerical
simulations under different scenarios on the same rock sample. The
simulations at pore-scale can then be interpretated to derive
macroscopic reservoir properties (e.g., permeability, formation factor,
elastic moduli, etc.) and used for sensitivity analysis. A
representative elementary volume (REV) of the rock sample with high
resolution is an important prerequisite for accurate digital rock
physics results. However, in practice, there is an inherent trade-off
between field of view (FoV) and image resolution due to the limitation
of imaging techniques (Wildenschild et al., 2002; Wildenschild and
Sheppard, 2013). Low-resolution imaging cannot resolve micro pores,
which often makes the estimated rock properties to be underestimated or
overestimated with regards to experimental measurements. On the other
hand, the high-resolution image may not capture a REV due to the small
FoV.

One way to overcome the trade-off is to integrate imaging data from
multiple sources, such as 3-D micro-CT images at the micron scale and
2-D SEM images at the nano scale. Traditional solutions to this
challenging problem are mainly stochastic methods based on the spatial
statistical information (e.g., two-point correlation functions and
multiple-point statistics). Jiao et al. (2007) modeled heterogeneous
materials from two-point correlation functions using simulated
annealing. Okabe and Blunt (2007) reconstructed 3-D pore space structure
by integrating micro-CT images at the micron-scale that resolve large
pores with statistically simulated high-resolution images from 2-D thin
sections that provides finer-scale features. Mohebi et al. (2009)
proposed a statistical method to fuse low-resolution measurements with a
high-resolution prior model. Tahmasebi et al. (2015) proposed a
multiscale and multiresolution reconstruction method to generate 3-D
models of shales using 2-D images.

In recent years, deep learning methods have been developed to alleviate
the trade-off between resolution and FoV. Wang et al. (2019) and Da Wang
et al. (2019) applied convolutional neural networks (CNN) for micro-CT
image enhancement. Da Wang et al. (2020) proposed a generative
adversarial network (GAN) to increase the micro-CT image resolution. The
above works are based on supervised learning methods that require a
large number of paired training data of corresponding low- and
high-resolution images. However, in practice, the paired training data
are often not available because the sample locations of imaging data are
often different. To circumvent this limitation, Niu et al., (2020)
proposed a cycle-in-cycle GAN to deal with the unpaired training data
for boosting lateral resolution of micro-CT images. You et al. (2021)
developed a progressive growing GAN to increase the vertical resolution
by combination with the technique of GAN inversion. The deep learning
methods are powerful for resolution enhancement of micro-CT images and
have high perceptive accuracy compared to traditional interpolation
algorithms (e.g., nearest neighborhood and bicubic interpolation).
However, previous works mainly focused on the super-resolution problem
aiming to increase the resolution of micro-CT images by two or four
times. They are useful to make the micro-CT images sharper but cannot
resolve pore structures at multiscale or integrate data from different
imaging modalities such as micro-CT and SEM images.

The major challenges in multiscale digital rock data fusion are (1) that
the high-resolution images are usually limited in number, which would
make the model to be easily overfitted and (2) that the digital rock
images from multiple sources are typically acquired at different
locations of the rock sample, which means that the training samples are
unpaired and therefore, supervised methods of machine learning are not
applicable. In this letter we propose an innovative method to solve this
problem based on deep neural networks. It uses a style-based GAN (Karras
et al., 2019; Karras et al., 2020a and 2020b) to augment the limited
high-resolution images and then fuses unpaired data at different
resolutions by a cycle-consistent GAN (CycleGAN) (Zhu et al., 2017). The
disentanglement representation learning of the style-based GAN allows us
to generate images with different styles by sampling in different
regions of the latent space. With such an advantage, we can train
multiple CycleGAN models by feeding training samples with different
styles and thus generate multiple high-resolution realizations of the
rock that are consistent with the input of low-resolution micro-CT.

\textbf{Methodology and Data}

The proposed workflow of multiscale digital rock data fusion is
illustrated in Figure 1. In this study, we have five carbonate samples,
most of which exhibit high heterogeneity and anisotropy. Each sample has
one high-resolution 2-D SEM image with a size of about 7500×4500 pixels
and two or three low-resolution 3-D micro-CT volumes with a size of
about 600×600×900 voxels. The resolutions of the micro-CT images range
from 1.0 to 2.0 \selectlanguage{greek}µ\selectlanguage{english}m, while the resolutions of all SEM images are 0.1 \selectlanguage{greek}µ\selectlanguage{english}m.
A Style-based GAN is first trained to augment the limited number of SEM
images and then a CycleGAN is used to reconstruct high-resolution images
from low-resolution micro-CT data.\selectlanguage{english}
\begin{figure}[H]
\begin{center}
\includegraphics[width=0.70\columnwidth]{figures/image1/image1}

\end{center}
\end{figure}

\textbf{Figure 1.} The overall workflow of multiscale digital rock data
fusion based on deep generative adversarial networks:
G\textsubscript{HR} is the generator of StyleGAN2 that takes the style
latent vector and random noise as input to simulate SEM images;
D\textsubscript{HR} is the discriminator of StyleGAN2 that aims to
distinguish between real SEM samples and fake samples generated by
G\textsubscript{HR}; G\textsubscript{L2H}and G\textsubscript{H2L} are
two generators of CycleGAN that predicts high-resolution image from
low-resolution micro-CT data and recovers low-resolution data from
high-resolution data, respectively; D\textsubscript{L} and
D\textsubscript{H} are two discriminators corresponding to
G\textsubscript{L2H} and G\textsubscript{H2L}, respectively.

\textbf{2.1 SEM Data Augmentation by StyleGAN2-ADA}

GANs are a class of generative network models typically consisting of
two competing players: a generator network \(G\) and a
discriminator network \(D\) (Goodfellow et al., 2014). The
generator \(G\) tries to synthesize fake samples
\(G(z)\) from the input random noise\(z\sim p_{z}(z)\) to fool
\(D\) by mimicking real samples\(x\sim p_{\text{data}}(x)\), while
\(D\) conversely aims to \selectlanguage{english}distinguish between real samples
\(x\sim p_{\text{data}}(x)\) and fake samples \(G(z)\). The notation
\(y\sim p(y)\) indicates that the random variable
\(y\) is distributed according to the probability
distribution\(p(y)\). The two networks \(G\)
and \(D\) are trained in an adversarial manner, which is
equivalent to playing a minimax game with a loss function
\(\mathcal{L}_{\text{GAN}}\left(D,G\right)\) given by

\begin{equation}
\operatorname{}{\operatorname{}{\mathcal{L}_{\text{GAN}}(D,G)}}=E_{x\sim p_{\text{data}}(x)}\left[\log{D\left(x\right)}\right]+E_{z\sim p_{z}(z)}\left[\log\left(1-D\left(G\left(z\right)\right)\right)\right].\ \ \ \ \ \ \ \ \ \ (1)\nonumber \\
 \end{equation}

With such an adversarial training scheme, GANs have the ability of
producing high quality sharp images, outperforming approaches based on
pixel-wise mean square error (MSE) loss (Goodfellow et al., 2014).

Karras et al. (2018, 2019) developed StyleGAN and StyleGAN2, two
variants of GANs, which are powerful in high-resolution images
generation. They can control not only the style (global features) of the
image at different scales but also stochastic details (local features).
Since StyleGANs are unsupervised, they are well-suited for datasets
without conditional labels, which are often common in real applications.
To address the long-standing challenge in GANs of training with small
dataset, (in which case the discriminator will quickly be overfit to the
training samples resulting in divergence of the training), Karras et al.
(2020) improved StyleGAN2 by introducing an adaptive discriminator
augmentation (ADA) mechanism, which is called StyleGAN2-ADA. With such
an augmentation mechanism, the style-based GANs perform well with
several thousand or even only several hundred training samples. A
detailed description of ADA can be found in the Supporting Information
S1.

StyleGAN2 consist of two components: a mapping network and a synthesis
network. The goal of the mapping network is to encode the input random
noise \(\mathbf{z}\mathbb{\in Z}\) into a set of intermediate
vectors\(\mathbf{w\in}\mathbb{W}\) using fully connected layers. Each
intermediate vector \(\mathbf{w}\) is further transformed to
produce a style scalar \(s\) by an affine transformation
\(\mathbf{A}\). The major benefit of the mapping network is that it
is helpful to disentangle the latent representation and therefore makes
our model easier to be interpreted. Then, the synthesis network
incorporates the styles \(s\)via a weight demodulation
operation to generate the artificial images starting from low resolution
(4\selectlanguage{ngerman}×4) and continuing to higher resolution (8×8, 16×16, \ldots{},
1024×1024) by convolutional layers. The shallower the layer in which the
style \(s\) is incorporated, the coarser the level of
details is affected. For example, the first few styles affect the coarse
level of details (4×4), while the last few styles affect the fine level
of details (1024×1024). This architecture enables the StyleGAN2 to
control the global features of images at different scales. To control
stochastic variations of generated images at different levels of
details, random noise after another affine
transformation\(\mathbf{B}\) is injected to the feature maps of the
convolutional blocks. It allows the generator to only change the local
features, leaving the overall styles and high-level details intact. A
detailed network architecture of StyleGAN2 can be found in the
Supporting Information Figure S2.

\textbf{2.2 Multiscale Digital Rock Images Fusion by CycleGAN}

Another challenge in fusing multiscale digital rock images is that the
images are acquired at different locations of the rock sample, which
means that the training samples are unpaired. For this reason, we adopt
a CycleGAN, which is an unsupervised method designed for image
cross-domain transfer (Zhu et al., 2017). Specifically, in this study,
we aim to transfer the micro-CT images from the low-resolution
domainLR to the high-resolution domain
HR by integrating the information from SEM data.

CycleGAN consists of four networks: (1) a generator \(G_{L2H}\)
mapping the images from domain LR to domain
HR, (2) a generator \(G_{H2L}\) mapping the
images from domain HR to domain LR, (3)
a discriminator \(D_{H}\) aiming to encourage\(G_{L2H}\)
to transfer LR into outputs \selectlanguage{english}indistinguishable from
domain HR and (4) a similar discriminator
\(D_{L}\) for\(G_{H2L}\). The major difference of
CycleGAN from traditional GANs is that it includes two generators to
constrain each other. It ensures that\(G_{L2H}\) outputs the
high-resolution image that is conditional to the input low-resolution
image and \selectlanguage{english}vice versa for \(G_{H2L}\). To achieve this goal,
apart from the adversarial loss (Eq. 1), we need to add a regularization
term to the loss function of the generators, namely the cycle-consistent
loss given by

\(\mathcal{L}_{\text{CYC}}\left(G_{L2H},\ G_{H2L}\right)=E_{x\sim p_{\text{LR}}\left(x\right)}\left[\left\|G_{H2L}\left(G_{L2H}\left(x\right)\right)\ -x\right\|_{1}\right]+E_{y\sim p_{\text{HR}}(y)}\left[\left\|G_{L2H}\left(G_{H2L}\left(y\right)\right)\ -y\right\|_{1}\right]\ \ (2)\).

The first term in Eq. 2 aims that \selectlanguage{english}for each image \(x\)
from domainLR, \selectlanguage{english}the image transfer cycle can transform
\(x\) back to the original input, i.e.,
\selectlanguage{english}\(G_{H2L}\left(G_{L2H}\left(x\right)\right)\ \approx x\). Similarly, \selectlanguage{english}for each image \(y\)
from domain HR, the second term in Eq. 2 ensures
backward cycle consistency:\(G_{L2H}\left(G_{H2L}\left(y\right)\right)\ \approx y\). A detailed network
architecture of CycleGAN can be found in the Supporting Information
Figure S3.

\textbf{2.3 Data Preparation and Training}

The training samples consist of two datasets: (1) low-resolution
sub-images of micro-CT with a size of 64\selectlanguage{ngerman}×64 and (2) high-resolution
sub-images of SEM with a size of 1024×1024. Considering that there are
only a limited number of real SEM images, we extract the high-resolution
sub-images with a 70\% overlap between sequential sub-images (with a
sliding stride of 500 pixels) to increase the number of training
samples. In total, there are 371 high-resolution samples of SEM. The
micro-CT data are sufficient in number, so we randomly extract the
low-resolution sub-images without any overlap. In total, there are
200,000 low-resolution samples of micro-CT images.

The training of the networks includes two steps. We start with the
StyleGAN2-ADA training for SEM data augmentation. It aims to learn the
underlying probability distribution (or manifold) where the real SEM
sub-images lie. The StyleGAN2-ADA is trained in parallel with four
Nvidia A100 GPUs each with 40 GB memory. It takes approximately 24 hours
to converge. After training, we use the generator of the StyleGAN2-ADA
to simulate 200,000 high-resolution sub-images to keep the number
comparable with the low-resolution images. Such data augmentation would
be helpful for the training of the following CycleGAN to avoid
overfitting. After training the CycleGAN, we can use the
generator\(G_{\text{LH}}\) to transfer the low-resolution images with a
size of 64×64 to the high-resolution domain with a size of 1024×1024.
The training of CycleGAN takes approximately 24 hours with one Nvidia
A100 GPU.

\textbf{Results}

Figures 2a and 2b show the high-resolution images extracted from the
real SEM data and those simulated by the trained StyleGAN2-ADA with
different global styles but constant noise as input, respectively. The
simulated images capture the fine details of microstructures in the SEM
data and are visually indistinguishable from the real samples. Moreover,
most pore types, i.e., inter- and intra-granular pore and microfracture
from the nano to micron scale, are accurately recovered. It indicates
that undesirable mode collapse does not occur in the training and the
trained StyleGAN2-ADA can guarantee the diversity of image generation.
To quantitively evaluate the quality of the generated images, we compute
the porosity, specific surface area and two-point correlation of both
the real and simulated images. As shown in Figure 3, the distributions
of the synthetic and real samples are consistent. It also indicates that
the microstructural details of the SEM data are well captured by the
StyleGAN2-ADA. The correlation curves shown in Figure 3c are close to
the exponential model defined as \(R\left(h\right)=e^{-h/\lambda}\)where
\(h\) is the lag distance and \(\lambda\) is the
correlation length. The correlation length \(\lambda\) of each
sample in Figure 3d is obtained by fitting the exponential model
\(R\left(h\right)\) to the two-point correlation curve. As can be seen
from the histograms of correlation lengths, the synthetic samples, while
broadly consistent with the true samples, do tend to have slightly
longer correlation lengths, indicating that some of the generated
synthetic samples may be smoother than the true images. Figure S4 in the
Supporting Information shows the synthetic images generated with fixed
styles but different noise. The generated samples look similar in global
features but different in local features and they are very close in
terms of the porosity, specific surface area and two-point correlation
(Figure S5 in the Supporting Information). The above different behaviors
with constant and changing global style vectors indicate that the
style-based GAN has a good ability of separating the global and local
styles underlying the training images. Thanks to the disentangled
representation of the latent space, we can generate new images by
interpolation in the latent style space. As shown in Figure S6 in the
Support Information, the generated images are smoothly transformed from
one end-member to another with progressive interpolation between the
latent space end-members.\selectlanguage{english}
\begin{figure}[H]
\begin{center}
\includegraphics[width=0.70\columnwidth]{figures/image2/image2}

\end{center}
\end{figure}

\textbf{Figure 2.} (a) real SEM images; (b) synthetic images generated
by the trained StyleGAN2-ADA with different global styles and constant
noise. The image size is 1024\selectlanguage{ngerman}×1024 and the resolution is 0.1 \selectlanguage{greek}µ\selectlanguage{english}m.\selectlanguage{english}
\begin{figure}[H]
\begin{center}
\includegraphics[width=0.70\columnwidth]{figures/image3/image3}

\end{center}
\end{figure}

\textbf{Figure 3.} \selectlanguage{english}Microstructure analyses for the real and synthetic
SEM samples: (a) porosity; (b) specific surface area; (c) two-point
correlation function; (d) two-point correlation length.

Figure 4a and 4b shows the predicted high-resolution images from
micro-CT data using the trained CycleGAN. With the generator
\(G_{L2H}\)of the trained CycleGAN, the low-resolution micro-CT
image with a size of 64\selectlanguage{ngerman}×64 can be increased by 16 times to the
high-resolution domain with a size of 1024×1024. We can see that the
reconstructed images not only keep the geometric structures of the input
micro-CT images but also recover the fine details present in the SEM
data. The recovered fine microstructural details would be helpful to
accurately predict pore-scale fluid flow and effective petrophysical
properties. Thanks to the disentanglement of the latent style space
learned by the StyleGAN2, we can control the styles of the synthetic SEM
images by sampling in different regions of the latent space. As
illustrated in the Supporting Information Figure S7, we can train
multiple CycleGAN models by feeding training samples with different
styles and thus generate multiple high-resolution realizations of the
rock that all are consistent with the input of low-resolution micro-CT.

Then, we use the CycleGAN to down-scale the entire 3-D micro-CT volumes.
The prediction is performed slice by slice and each slice is divided
into patches with a size of 64×64. The vertical resolution of micro-CT
is first increased by 16 times by bicubic interpolation to make the
reconstructed rock models have same resolution along all three
directions. To mitigate the artifacts at boundaries, there is an overlap
of 8 pixels between patches. One predicted slice with a larger size of
3584×3584 is shown in the Supporting Information Figure S8. The
predicted high-resolution images are close to the true SEM image from
the same rock sample in terms of porosity, specific surface area and
two-point correlation as illustrated in the Supporting Information
Figure S9. Figure 4c and 4d show the pore-scale Stokes flow (\selectlanguage{english}the slow,
incompressible, viscous steady flow) (Allen, 2021) simulated by the LIR
solver of the GeoDict software (Linden et al., 2015) over a sub-cube of
micro-CT with a size of 64\selectlanguage{ngerman}×64×64 and the reconstructed high-resolution
rock, respectively. We can observe more details from the reconstructed
high-resolution model and thus have deeper understanding of physical
procedures at pore-scale. Figure 4f shows one 3-D realization with a
size of 1792×1792×2688 from the micro-CT data with a size of 112×112×168
(Figure 4e). More realizations can be found in the Supporting
Information Figure S10. As shown in Table 1, the predicted
permeabilities of the reconstructed high-resolution models are more
consistent with the permeability from the laboratory core measurement
than the prediction from micro-CT data.\selectlanguage{english}
\begin{figure}[H]
\begin{center}
\includegraphics[width=0.70\columnwidth]{figures/image4/image4}

\end{center}
\end{figure}

\textbf{Figure 4.} (a) micro-CT images (64\selectlanguage{ngerman}×64; dx=1.6 \selectlanguage{greek}µ\selectlanguage{english}m) and (b)
prediction of CycleGAN (1024\selectlanguage{ngerman}×1024; dx=0.1 \selectlanguage{greek}µ\selectlanguage{english}m); (c) velocity field of
fluid flow simulation on the micro-CT sub-cube (64\selectlanguage{ngerman}×64); (d) velocity
field of the fluid flow simulation on the reconstructed model at the SEM
scale (64×64); (e) input micro-CT volume (112×112×168); (f)
reconstructed high-resolution model (1792×1792×2688).

Table 1. Porosity and permeability of the low-resolution micro-CT and
reconstructed high-resolution realizations (M1-M7) as well as the lab
measurement.\selectlanguage{english}
\begin{longtable}[]{@{}llllllllll@{}}
\toprule
& Lab & micro-CT & M1 & M2 & M3 & M4 & M5 & M6 & M7\tabularnewline
\midrule
\endhead
Porosity & 0.19 & 0.19 & 0.17 & 0.16 & 0.15 & 0.13 & 0.23 & 0.10 &
0.14\tabularnewline
Permeability (mD) & 4.0 & 132.75 & 1.79 & 3.98 & 1.92 & 4.05 & 8.70 &
1.06 & 2.50\tabularnewline
\bottomrule
\end{longtable}

\textbf{Discussion}In recent years, attempts have been made to overcome
the trade-off between resolution and FoV for digital rock data using
deep learning methods (e.g., CNNs and GANs). However, previous works
focused on the super-resolution problem aiming to increase the
resolution of micro-CT images by two or four times. They are useful to
make the micro-CT images sharper but cannot resolve pore structures at
multiscale. Our proposed method provides an effective means to
reconstruct rock models that accurately capture multiscale pore
structures obtained by different imaging methods (e.g., micro-CT and
SEM). Due to the limitation of GPU memory and non-availability of 3-D
SEM data, the output high-resolution image is only in 2D and with a size
of 1024\selectlanguage{ngerman}×1024 and therefore, we need to perform the prediction
patch-by-patch by dividing the micro-CT slice into sub-images. The
straightforward solution to increase the image size is to use more GPUs,
but a more efficient way is to decrease the number of network parameters
by model compression (e.g., depthwise separable convolutions, network
pruning and knowledge distillation). The potential solution to 3-D
simulation is to develop a GAN with the generator in 3-D while the
discriminator in 2-D which takes the slices sampled from generated
synthetic 3-D images and real SEM data. Those are the research
directions that we will investigate in future.

\textbf{Conclusion}

We presented an innovative approach for fusion of multiscale digital
rock images, i.e., low-resolution micro-CT and high-resolution SEM data,
using StyleGAN2-ADA and CycleGAN. The StyleGAN2-ADA network is effective
to overcome the issue of overfitting due to limited number of SEM
images, while the CycleGAN network allows for leveraging unpaired
training samples of micro-CT and SEM, which is a common challenge in
practice. The application to a carbonate dataset reveals that the
proposed methodology is a valid and powerful approach for integrating
multiscale digital rock data. The reconstructed rock models accurately
capture the micro-structures from both low-resolution micro-CT and
high-resolution SEM images. Moreover, the computed effective
permeabilities are more accurate than the prediction directly from
micro-CT data by comparison with the laboratory measurement. We conclude
that the proposed method provides an efficient means to reconstruct
high-resolution digital rocks with large FoV, which is of great
significance for the accurate pore-scale flow simulation and
petrophysical properties prediction.

\textbf{Data Availability Statement}

The code is freely available on the GitHub repository
(https://github.com/theanswer003/ MultiscaleDRPNet).

\textbf{Acknowledgements}

\selectlanguage{english}We acknowledge Shell for  \selectlanguage{english}financial support and for providing the
digital rock images. This study has mostly been performed using the
Sherlock cluster at Stanford University. We are grateful to Stanford
University and the Stanford Research Computing Center for providing
computational resources and support in this research. We acknowledge the
sponsors of the Stanford Center for Earth Resources Forecasting (SCERF)
and support from Prof. Steve Graham, the Dean of the Stanford School of
Earth, Energy and Environmental Sciences. The authors also would like to
thank Math2Market for providing the GeoDict software.

\textbf{References}

\begin{itemize}
\tightlist
\item
  Allen, M.B., 2021. The Mathematics of Fluid Flow in Porous Media, John
  Wiley \& Sons, Hoboken, NJ.
\item
  Andr\selectlanguage{ngerman}ä, H., Combaret, N., Dvorkin, J., Glatt, E., Han, J., Kabel, M.,
  Keehm, Y., Krzikalla, F., Lee, M., Madonna, C. and Marsh, M., 2013a.
  Digital rock physics benchmarks---Part I: Imaging and segmentation.
  Computers \& Geosciences, 50, pp.25-32.
\item
  Andrä, H., Combaret, N., Dvorkin, J., Glatt, E., Han, J., Kabel, M.,
  Keehm, Y., Krzikalla, F., Lee, M., Madonna, C. and Marsh, M., 2013b.
  Digital rock physics benchmarks---Part II: Computing effective
  properties. Computers \& Geosciences, 50, pp.33-43.
\item
  Blunt, M.J., Bijeljic, B., Dong, H., Gharbi, O., Iglauer, S.,
  Mostaghimi, P., Paluszny, A. and Pentland, C., 2013. Pore-scale
  imaging and modelling. Advances in Water resources, 51, pp.197-216.
\item
  Da Wang, Y., Armstrong, R.T. and Mostaghimi, P., 2019. Enhancing
  resolution of digital rock images with super resolution convolutional
  neural networks. Journal of Petroleum Science and Engineering, 182,
  p.106261.
\item
  Da Wang, Y., Armstrong, R.T. and Mostaghimi, P., 2020. Boosting
  resolution and recovering texture of 2D and 3D micro\selectlanguage{english}-CT images with
  deep learning. Water Resources Research, 56(1), p.e2019WR026052.
\item
  Karras, T., Laine, S. and Aila, T., 2019. A style-based generator
  architecture for generative adversarial networks. In Proceedings of
  the IEEE/CVF Conference on Computer Vision and Pattern Recognition
  (pp. 4401-4410).
\item
  Karras, T., Laine, S., Aittala, M., Hellsten, J., Lehtinen, J. and
  Aila, T., 2020a. Analyzing and improving the image quality of
  stylegan. In Proceedings of the IEEE/CVF Conference on Computer Vision
  and Pattern Recognition (pp. 8110-8119).
\item
  Karras, T., Aittala, M., Hellsten, J., Laine, S., Lehtinen, J. and
  Aila, T., 2020b. Training generative adversarial networks with limited
  data. arXiv preprint arXiv:2006.06676.
\item
  Keehm, Y., Mukerji, T. and Nur, A., 2001. Computational rock physics
  at the pore scale: Transport properties and diagenesis in realistic
  pore geometries. The Leading Edge, 20(2), pp.180-183.
\item
  Keehm, Y., Mukerji, T. and Nur, A., 2004. Permeability prediction from
  thin sections: 3D reconstruction and Lattice\selectlanguage{english}-Boltzmann flow
  simulation. Geophysical Research Letters, 31(4).
\item
  Linden, S., Wiegmann, A. and Hagen, H., 2015. The LIR space
  partitioning system applied to the Stokes equations. Graphical Models,
  82, pp.58-66.
\item
  Mohebi, A., Fieguth, P. and Ioannidis, M.A., 2009. Statistical fusion
  of two-scale images of porous media. Advances in water resources,
  32(11), pp.1567-1579.
\item
  Niu, Y., Wang, Y.D., Mostaghimi, P., Swietojanski, P. and Armstrong,
  R.T., 2020. An innovative application of generative adversarial
  networks for physically accurate rock images with an unprecedented
  field of view. Geophysical Research Letters, 47(23), p.e2020GL089029.
\item
  Jiao, Y., Stillinger, F.H. and Torquato, S., 2007. Modeling
  heterogeneous materials via two-point correlation functions: Basic
  principles. Physical review E, 76(3), p.031110.
\item
  Okabe, H. and Blunt, M.J., 2007. Pore space reconstruction of vuggy
  carbonates using microtomography and multiple\selectlanguage{english}-point statistics. Water
  Resources Research, 43(12).
\item
  Saxena, N., Hofmann, R., Alpak, F.O., Berg, S., Dietderich, J.,
  Agarwal, U., Tandon, K., Hunter, S., Freeman, J. and Wilson, O.B.,
  2017. References and benchmarks for pore-scale flow simulated using
  micro-CT images of porous media and digital rocks. Advances in Water
  Resources, 109, pp.211-235.
\item
  Tahmasebi, P., Javadpour, F. and Sahimi, M., 2015. Multiscale and
  multiresolution modeling of shales and their flow and morphological
  properties. Scientific reports, 5(1), pp.1-11.
\item
  Wang, Y., Teng, Q., He, X., Feng, J. and Zhang, T., 2019. CT-image of
  rock samples super resolution using 3D convolutional neural network.
  Computers \& Geosciences, 133, p.104314.
\item
  Wildenschild, D., Vaz, C.M.P., Rivers, M.L., Rikard, D. and
  Christensen, B.S.B., 2002. Using X-ray computed tomography in
  hydrology: systems, resolutions, and limitations. Journal of
  Hydrology, 267(3-4), pp.285-297.
\item
  Wildenschild, D. and Sheppard, A.P., 2013. X-ray imaging and analysis
  techniques for quantifying pore-scale structure and processes in
  subsurface porous medium systems. Advances in Water resources, 51,
  pp.217-246.
\item
  You, N., Li, Y.E. and Cheng, A., 2021. 3D Carbonate Digital Rock
  Reconstruction Using Progressive Growing GAN. Journal of Geophysical
  Research: Solid Earth, 126(5), p.e2021JB021687.
\item
  Zhu, J.Y., Park, T., Isola, P. and Efros, A.A., 2017. Unpaired
  image-to-image translation using cycle-consistent adversarial
  networks. In Proceedings of the IEEE international conference on
  computer vision (pp. 2223-2232).
\end{itemize}

\textbf{Hosted file}

\verb`gl098342_si.docx`  available at \url{https://authorea.com/users/545153/articles/601933-multiscale-fusion-of-digital-rock-images-based-on-deep-generative-adversarial-networks}

\selectlanguage{english}
\FloatBarrier
\end{document}

