diff --git a/handout/diagrams/area_sampling.tex b/handout/diagrams/area_sampling.tex index 37c1979..0ee8968 100644 --- a/handout/diagrams/area_sampling.tex +++ b/handout/diagrams/area_sampling.tex @@ -21,7 +21,7 @@ % occlusion tri \draw[ fill, - red + darkgray ] (3,2) -- (4.5,0.2) -- (6,3); % samples @@ -33,5 +33,5 @@ 5/0.5, 5/2.5 } - \node[fill = yellow] at (\x,\y) {}; + \node[fill = black] at (\x,\y) {}; \end{tikzpicture} \ No newline at end of file diff --git a/handout/diagrams/diagramm_occlusion.tex b/handout/diagrams/diagramm_occlusion.tex index 043319c..1898c20 100644 --- a/handout/diagrams/diagramm_occlusion.tex +++ b/handout/diagrams/diagramm_occlusion.tex @@ -1,24 +1,24 @@ \begin{tikzpicture}[scale = 0.9, every node/.style={scale=0.9}] - \def\raynum{60} + \def\raynum{18} \foreach \x in {0,...,\raynum} { - \ifthenelse{\x < 10 \OR \x>50} + \ifthenelse{\x < 3 \OR \x>15} { - \draw[color = darkgray] (\x /\raynum * 6 ,0) -- (3,4); + \draw[color = black] (\x /\raynum * 6 ,0) -- (3,4); }{ - \draw[color = red] (\x /\raynum * 6 ,0) -- (3,4); + \draw[color = lightgray] (\x /\raynum * 6 ,0) -- (3,4); } } \fill[ white - ] (1.49,1) -- (4.51,1) -- (5.01,0) -- (0.99,0) -- cycle; + ] (1.48,1) -- (4.52,1) -- (5.02,0) -- (0.98,0) -- cycle; - \draw [color=darkgray] (0,0) -- node[below]{wall} (6,0) ; % wall - \draw [color=red](1.5,1) -- node[below]{blocker} (4.5,1); %blocker + \draw [color=black] (0,0) -- node[below]{wall} (6,0) ; % wall + \draw [color=black](1.5,1) -- node[below]{blocker} (4.5,1); %blocker %cam \node[ @@ -37,11 +37,11 @@ \fill[ - red + gray ] (5,3) -- (6.7,3) -- (6.7,3.3) -- (5,3.3) -- cycle; - \draw[darkgray] (8,2) coordinate (darkgray_ray_isec) -- (5,5) coordinate (orig); - \draw[red] (6.4,3.3) coordinate (red_ray_isec) -- (5,5); + \draw[black] (8,2) coordinate (darkgray_ray_isec) -- (5,5) coordinate (orig); + \draw[gray] (6.4,3.3) coordinate (red_ray_isec) -- (5,5); %line \draw (mag.north east) -- (7,2); diff --git a/handout/diagrams/edge_sampling.tex b/handout/diagrams/edge_sampling.tex index 826c0be..3fb0cd1 100644 --- a/handout/diagrams/edge_sampling.tex +++ b/handout/diagrams/edge_sampling.tex @@ -22,7 +22,7 @@ % occlusion tri \draw[ fill, - red + darkgray ] (3,2) -- (4.5,0.2) -- (6,3); % samples @@ -33,10 +33,10 @@ 3.5/2.5, 5.5/0.5 } - \node[fill = yellow] at (\x,\y) {}; + \node[fill = black] at (\x,\y) {}; \node[ - fill = yellow, - fill opacity = 0.4, + fill = black, + fill opacity = 0.5, ] at (4.5,1.5) {}; \end{tikzpicture} diff --git a/handout/diagrams/geometry_term_diff.tex b/handout/diagrams/geometry_term_diff.tex index e8589e0..af9360d 100644 --- a/handout/diagrams/geometry_term_diff.tex +++ b/handout/diagrams/geometry_term_diff.tex @@ -5,8 +5,8 @@ \draw[->] (-0.2,-0.5) -- (2.2,-0.5) node[right] {$\omega$}; \draw[->] (0,-0.7) -- (0,2.2) node[above] {$V(x,\omega)$}; - \draw (0,0) -- (1,0) plot coordinates {(0,0) (1,0)}[color=red]; - \draw (1,1) -- (2,1) plot coordinates {(1,1) (2,1)}[color=red]; + \draw (0,0) -- (1,0) plot coordinates {(0,0) (1,0)}[color=black]; + \draw (1,1) -- (2,1) plot coordinates {(1,1) (2,1)}[color=black]; \draw (0,0) node[left] {$0$}; \draw (0,1) node[left] {$1$}; \draw (1,-0.7) node[below] {$\omega_0$}; @@ -21,11 +21,11 @@ \draw[->] (-0.2,-0.5) -- (2.2,-0.5) node[right] {$\omega$}; \draw[->] (0,-0.7) -- (0,2.2) node[above] {$\frac{\partial}{\partial\omega}V(x,\omega)$}; - \draw (0,0) -- (2,0) plot coordinates {(0,0) (2,0)}[color=red]; + \draw (0,0) -- (2,0) plot coordinates {(0,0) (2,0)}[color=black]; \draw (0,0) node[left] {$0$}; \draw (0,1) node[left] {$1$}; \draw (0,2) node[left] {$\infty$}; - \draw (1,2) node[color=red] {$\bullet$}; + \draw (1,2) node[color=black] {$\bullet$}; \draw (1,-0.7) node[below] {$\omega_0$}; \end{tikzpicture} } \ No newline at end of file diff --git a/handout/diagrams/halfspaces.tex b/handout/diagrams/halfspaces.tex index 580d2bb..1675e5f 100644 --- a/handout/diagrams/halfspaces.tex +++ b/handout/diagrams/halfspaces.tex @@ -7,7 +7,7 @@ \draw[ fill, - red + gray ] (-2, -3) -- (-2,-4) -- (2,-6) -- (4.3,-6) -- (4.3,-4) -- cycle; \draw[ @@ -17,12 +17,12 @@ \draw[ fill, - blue + darkgray ] (1.7,-2) circle (15pt); \draw[ fill, - blue + darkgray ] (0.4,-4.2) circle (15pt); \node (fu) at (1.4,-1) {\LARGE $f_u$ $(+)$}; diff --git a/handout/handout.tex b/handout/handout.tex index 016894e..400a560 100644 --- a/handout/handout.tex +++ b/handout/handout.tex @@ -1,4 +1,3 @@ -%\newcommand{\CLASSINPUTtoptextmargin}{1.5cm} \newcommand{\CLASSINPUTbottomtextmargin}{2.5cm} \documentclass[conference, a4paper, hidelinks]{IEEEtran} @@ -37,6 +36,7 @@ \usepackage{amsfonts} \usepackage{caption} \DeclareCaptionType{eqcap}[Equation][List of equations] +\usepackage[T1]{fontenc} \usepackage{tikz} \usetikzlibrary{calc, @@ -143,13 +143,13 @@ Given this function and a set of target images the inverse rendering problem can \begin{figure}[h] \centering \subfloat[initial guess]{ - \includegraphics[width=0.3\linewidth]{../presentation/img/results/guess.png} + \includegraphics[width=0.3\linewidth]{../presentation/img/results/guess-bw.png} } \subfloat[optimized result]{ - \includegraphics[width=0.3\linewidth]{../presentation/img/results/result.png}\label{fig:result_image} + \includegraphics[width=0.3\linewidth]{../presentation/img/results/result-bw.png}\label{fig:result_image} } \subfloat[target]{ - \includegraphics[width=0.3\linewidth]{../presentation/img/results/photo.png} + \includegraphics[width=0.3\linewidth]{../presentation/img/results/photo-bw.png} } \caption{A generic example for how differentiable ray tracing can be used to approximate a solution for the inverse rendering problem.}\label{fig:inverse_rendering_example} \end{figure} diff --git a/handout/handout_bw.pdf b/handout/handout_bw.pdf new file mode 100644 index 0000000..474940a Binary files /dev/null and b/handout/handout_bw.pdf differ diff --git a/presentation/diagrams/raytracing_anim.tex b/presentation/diagrams/raytracing_anim.tex index 7108a5c..92def99 100644 --- a/presentation/diagrams/raytracing_anim.tex +++ b/presentation/diagrams/raytracing_anim.tex @@ -41,7 +41,7 @@ \begin{tikzpicture} % scene seite \node[above] at (2,2) {side view}; - \node[below] at (2,-0.5) {(no indirect lighting)}; + \node[below] at (2,-0.5) {\footnotesize (no indirect lighting)}; \draw[ kit-green100 diff --git a/presentation/img/results/guess-bw.png b/presentation/img/results/guess-bw.png new file mode 100644 index 0000000..12b06d7 Binary files /dev/null and b/presentation/img/results/guess-bw.png differ diff --git a/presentation/img/results/photo-bw.png b/presentation/img/results/photo-bw.png new file mode 100644 index 0000000..4acd07a Binary files /dev/null and b/presentation/img/results/photo-bw.png differ diff --git a/presentation/img/results/result-bw.png b/presentation/img/results/result-bw.png new file mode 100644 index 0000000..7f8a0a8 Binary files /dev/null and b/presentation/img/results/result-bw.png differ diff --git a/presentation/modules/basic_terms.tex b/presentation/modules/basic_terms.tex index 19a9c9a..64d8866 100644 --- a/presentation/modules/basic_terms.tex +++ b/presentation/modules/basic_terms.tex @@ -68,10 +68,10 @@ \input{diagrams/raytracing_anim} \end{frame} -\begin{frame}{Image synthesis~-~Optical Phenomena} - \centering - \includegraphics[width=0.38\linewidth]{proseminar_cycles_annotated.png} -\end{frame} +%\begin{frame}{Image synthesis~-~Optical Phenomena} +% \centering +% \includegraphics[width=0.38\linewidth]{proseminar_cycles_annotated.png} +%\end{frame} \subsection{Differentiable Rendering} \begin{frame}{Differentiable Rendering} diff --git a/presentation/modules/motivation.tex b/presentation/modules/motivation.tex index 49d6833..82de314 100644 --- a/presentation/modules/motivation.tex +++ b/presentation/modules/motivation.tex @@ -65,6 +65,7 @@ with Differentiable Monte Carlo Raytracing [\cite{ACM:inverse_rendering}]\\ \end{itemize} \pause{} \vspace{15mm} + \footnotesize Image source: Auth0, \href{https://auth0.com/blog/captcha-can-ruin-your-ux-here-s-how-to-use-it-right/}{CAPTCHA Can Ruin Your UX. Here’s How to Use it Right} \end{minipage} \begin{minipage}{0.5\linewidth} diff --git a/presentation/modules/problems.tex b/presentation/modules/problems.tex index 98a79b7..c3d393f 100644 --- a/presentation/modules/problems.tex +++ b/presentation/modules/problems.tex @@ -1,5 +1,5 @@ \section{Problems} -% \begin{frame} +% \begin{frame} % \centering % \Huge % Problems diff --git a/presentation/modules/this_method.tex b/presentation/modules/this_method.tex index 5d668de..d9697d0 100644 --- a/presentation/modules/this_method.tex +++ b/presentation/modules/this_method.tex @@ -9,9 +9,9 @@ \setbeamercovered{transparent} \begin{block}{Assumptions} \begin{itemize} - \item Continuous parameter set \item Triangle meshes \item No interpenetrating triangles + \item Continuous parameter set \item No point lights, no perfectly specular surfaces \item Ignore time domain \end{itemize} @@ -66,8 +66,8 @@ \[ \sum_i\iint \underbrace{\theta(}_{\text{Step function}} - \underbrace{\alpha_i(x,y)}_{\text{Edge equation formed by the triangle edge}}) - \underbrace{f_i(x,y)}_{\text{Half space (may contain Heaviside step functions itself)}} + \underbrace{\alpha_i(x,y)}_{\text{Edge equation}}) + \underbrace{f_i(x,y)}_{\text{Half space}} \] } @@ -87,47 +87,66 @@ \end{frame} -\begin{frame}{Inverse Rendering~-~Results in this Paper} - \begin{block}{Inverse rendering here} - \begin{itemize} - \item Parameters: Camera pose, material parameters, light source intensity - \item Scene: Strong indirect illumination and non lambertian materials - \item Initial guess: Almost all objects white, arbitrary camera pose - \item 177 parameters - \item Loss function: Absolute difference - \item ADAM optimizer - \item Start resolution of $64\times 64$, linearly increase to $512\times 512$ in 8 steps\\ - $\implies$ Avoid local minima of the loss function - \end{itemize} - \end{block} +\begin{frame}{Inverse Rendering~-~Example from this Paper} + \centering + \begin{minipage}{0.19\linewidth} + \begin{figure} + \centering + \includegraphics[width=\linewidth]{img/teapot_video/teapot_init.png} + \vspace{0mm} + \caption{initial guess}\label{fig:teapot_init} + \end{figure} + \end{minipage} + \begin{minipage}{0.19\linewidth} + \begin{figure} + \centering + \includegraphics[width=\linewidth]{img/teapot_video/teapot_init_diff.png} + \caption{difference\\ + initial $\leftrightarrow$ target}\label{fig:teapot_init_diff} + \end{figure} + \end{minipage} + \begin{minipage}{0.19\linewidth} + \begin{figure} + \centering + \includegraphics[width=\linewidth]{img/teapot_video/teapot_target.png} + \vspace{0mm} + \caption{target image}\label{fig:teapot_target} + \end{figure} + \end{minipage} + \begin{minipage}{0.19\linewidth} + \begin{figure} + \centering + \includegraphics[width=\linewidth]{img/teapot_video/teapot_final_diff.png} + \caption{difference\\ + final $\leftrightarrow$ target}\label{fig:teapot_final_diff} + \end{figure} + \end{minipage} + \begin{minipage}{0.19\linewidth} + \begin{figure} + \centering + \includegraphics[width=\linewidth]{img/teapot_video/teapot_final.png} + \vspace{0mm} + \caption{final image}\label{fig:teapot_final} + \end{figure} + \end{minipage} \end{frame} -\begin{frame}{Inverse Rendering~-~Results in this Paper} -\begin{center} - \begin{minipage}{0.25\linewidth} - \begin{figure} +\begin{frame}{Inverse Rendering~-~Example from this Paper} \centering - \includegraphics[width=\linewidth]{img/results/guess.png} - \caption{Initial guess}\label{fig:results-guess} - \end{figure} - \end{minipage} - \hspace{2mm} - \begin{minipage}{0.25\linewidth} - \begin{figure} - \centering - \includegraphics[width=\linewidth]{img/results/photo.png} - \caption{Target (photograph)}\label{fig:results-target} - \end{figure} - \end{minipage} - \hspace{2mm} - \begin{minipage}{0.25\linewidth} - \begin{figure} - \centering - \includegraphics[width=\linewidth]{img/results/result.png} - \caption{Optimized image}\label{fig:results-optimized} - \end{figure} - \end{minipage} -\end{center} + \includemedia[ + width=0.62\linewidth,height=0.35\linewidth, + activate=onclick, + addresource=teapot.mp4, + playbutton=fancy, + transparent, + passcontext, + flashvars={ + source=teapot.mp4 + &autoPlay=true + } + ]{}{VPlayer.swf} + \\ + All media in this section taken from~\cite{ACM:diffable_raytracing} \end{frame} \begin{frame} @@ -231,66 +250,47 @@ \end{figure} \end{frame} -\begin{frame}{Inverse Rendering~-~Example from this Paper} - \centering - \begin{minipage}{0.19\linewidth} - \begin{figure} - \centering - \includegraphics[width=\linewidth]{img/teapot_video/teapot_init.png} - \vspace{0mm} - \caption{initial guess}\label{fig:teapot_init} - \end{figure} - \end{minipage} - \begin{minipage}{0.19\linewidth} - \begin{figure} - \centering - \includegraphics[width=\linewidth]{img/teapot_video/teapot_init_diff.png} - \caption{difference\\ - initial $\leftrightarrow$ target}\label{fig:teapot_init_diff} - \end{figure} - \end{minipage} - \begin{minipage}{0.19\linewidth} - \begin{figure} - \centering - \includegraphics[width=\linewidth]{img/teapot_video/teapot_target.png} - \vspace{0mm} - \caption{target image}\label{fig:teapot_target} - \end{figure} - \end{minipage} - \begin{minipage}{0.19\linewidth} - \begin{figure} - \centering - \includegraphics[width=\linewidth]{img/teapot_video/teapot_final_diff.png} - \caption{difference\\ - final $\leftrightarrow$ target}\label{fig:teapot_final_diff} - \end{figure} - \end{minipage} - \begin{minipage}{0.19\linewidth} - \begin{figure} - \centering - \includegraphics[width=\linewidth]{img/teapot_video/teapot_final.png} - \vspace{0mm} - \caption{final image}\label{fig:teapot_final} - \end{figure} - \end{minipage} +\begin{frame}{Inverse Rendering~-~Results in this Paper} + \begin{block}{Inverse rendering here} + \begin{itemize} + \item Parameters: Camera pose, material parameters, light source intensity + \item Scene: Strong indirect illumination and non lambertian materials + \item Initial guess: Almost all objects white, arbitrary camera pose + \item 177 parameters + \item Loss function: Absolute difference + \item ADAM optimizer + \item Start resolution of $64\times 64$, linearly increase to $512\times 512$ in 8 steps\\ + $\implies$ Avoid local minima of the loss function + \end{itemize} + \end{block} \end{frame} -\begin{frame}{Inverse Rendering~-~Example from this Paper} +\begin{frame}{Inverse Rendering~-~Results in this Paper} +\begin{center} + \begin{minipage}{0.25\linewidth} + \begin{figure} \centering - \includemedia[ - width=0.62\linewidth,height=0.35\linewidth, - activate=onclick, - addresource=teapot.mp4, - playbutton=fancy, - transparent, - passcontext, - flashvars={ - source=teapot.mp4 - &autoPlay=true - } - ]{}{VPlayer.swf} - \\ - All media in this section taken from~\cite{ACM:diffable_raytracing} + \includegraphics[width=\linewidth]{img/results/guess.png} + \caption{Initial guess}\label{fig:results-guess} + \end{figure} + \end{minipage} + \hspace{2mm} + \begin{minipage}{0.25\linewidth} + \begin{figure} + \centering + \includegraphics[width=\linewidth]{img/results/photo.png} + \caption{Target (photograph)}\label{fig:results-target} + \end{figure} + \end{minipage} + \hspace{2mm} + \begin{minipage}{0.25\linewidth} + \begin{figure} + \centering + \includegraphics[width=\linewidth]{img/results/result.png} + \caption{Optimized image}\label{fig:results-optimized} + \end{figure} + \end{minipage} +\end{center} \end{frame} \subsection{Conclusion~-~What can this Method do?} diff --git a/presentation/notes.md b/presentation/notes.md new file mode 100644 index 0000000..722b254 --- /dev/null +++ b/presentation/notes.md @@ -0,0 +1,58 @@ +# Notes diffable Monte carlo RT + +## Raytracing formula + +- geometry term discussed later +- Emission + All light reflected towards point +- Yields infinite recursion -> not calculable + +## Visualization + +- Explain image +- No indirect lighting! +- Output image is what we would expect (explain shade) + +## Differentiable rendering + +- That function is dependent on renderer +- Renderer needs to be differentiable + +## Importance + +- Inversely render complex indoor scenes +- "Fool" neural network +- Real time realistic shading in AR +- Application in maritime research + +## Adversarial image generation + +- Example for classification on slide 2! +- Fool neural netweork into wrongly classifying input data + - Optimize Image into wrong class + +## Why differentiable rendering is hard + +- Example later +- geometry term explanation later + +## Former methods visualization + +- Plane lit by a point light source. +- gradient with respect to the plane moving right +- light source remains static => the gradient should only be $\ne 0$ at the boundaries +- OpenDR and Neural not able to correctly calculate the gradients + - they are based on color buffer differences + +## Edge sampling + +- Approximate point lights using small area lights +- Specular => angle of incidence = angle of light reflected +- only lambertian materials + +## Edge Sampling - Math Background + +- Heaviside step functions in $f_i(x,y)$ + +## Inverse Rendering - Results in this paper + +- ADAM: talk by Mr. Wu \ No newline at end of file diff --git a/presentation/notes.pdf b/presentation/notes.pdf new file mode 100644 index 0000000..e0adaf4 Binary files /dev/null and b/presentation/notes.pdf differ diff --git a/presentation/presentation.tex b/presentation/presentation.tex index 5759353..f4008f8 100644 --- a/presentation/presentation.tex +++ b/presentation/presentation.tex @@ -16,7 +16,7 @@ \titleimage{logo_rt.pdf} -\newcommand{\presentationdate}{\emph{set date here}} +\newcommand{\presentationdate}{July 26 2023} %% Gruppenlogo \grouplogo{} @@ -27,7 +27,7 @@ % Beginn der Präsentation -\title[Differentiable Monte Carlo Ray Tracing through Edge Sampling]{Differentiable Monte Carlo Ray Tracing through Edge Sampling} +\title[Differentiable Monte Carlo Ray Tracing through Edge Sampling]{Differentiable Monte Carlo Ray Tracing through Edge Sampling [\cite{ACM:diffable_raytracing}]} \subtitle{Presentation for the Proseminar ``Differentiable Programming``} \author[Clemens Dautermann]{Clemens Dautermann}