@article{huang_normalization_2023,
  title = {Normalization {Techniques} in {Training} {DNNs}: {Methodology}, {Analysis} and {Application}},
  volume = {45},
  issn = {1939-3539},
  shorttitle = {Normalization {Techniques} in {Training} {DNNs}},
  doi = {10.1109/TPAMI.2023.3250241},
  number = {8},
  journal = {IEEE Transactions on Pattern Analysis and Machine Intelligence},
  author = {Huang, Lei and Qin, Jie and Zhou, Yi and Zhu, Fan and Liu, Li and Shao, Ling},
  month = aug,
  year = {2023},
  note = {Conference Name: IEEE Transactions on Pattern Analysis and Machine Intelligence},
  keywords = {Batch normalization, Biological neural networks, Covariance matrices, Decorrelation, deep neural networks, image classification, Optimization, survey, Task analysis, Tensors, Training, weight normalization},
  pages = {10173--10196},
  arxiv = {arXiv:2009.12836 [cs, stat]},
  arxivdoi = {https://doi.org/10.48550/arXiv.2009.12836},
  freepdf = {https://arxiv.org/pdf/2009.12836.pdf},
  tldr = {A review and commentary of normalization methods in DNNs. I skimmed this one.  Good for definitions of all of the normalization terms and especially Figure 1.}
}

Normalization techniques are essential for accelerating the training and improving the generalization of deep neural networks (DNNs), and have successfully been used in various applications. This paper reviews and comments on the past, present and future of normalization methods in the context of DNN training. We provide a unified picture of the main motivation behind different approaches from the perspective of optimization, and present a taxonomy for understanding the similarities and differences between them. Specifically, we decompose the pipeline of the most representative normalizing activation methods into three components: the normalization area partitioning, normalization operation and normalization representation recovery. In doing so, we provide insight for designing new normalization technique. Finally, we discuss the current progress in understanding normalization methods, and provide a comprehensive review of the applications of normalization for particular tasks, in which it can effectively solve the key issues.

@article{kessy_optimal_2018,
  title = {Optimal {Whitening} and {Decorrelation}},
  volume = {72},
  issn = {0003-1305},
  url = {https://doi.org/10.1080/00031305.2016.1277159},
  doi = {10.1080/00031305.2016.1277159},
  number = {4},
  urldate = {2023-09-08},
  journal = {The American Statistician},
  author = {Kessy, Agnan and Lewin, Alex and Strimmer, Korbinian},
  month = oct,
  year = {2018},
  note = {Publisher: Taylor \& Francis
  \_eprint: https://doi.org/10.1080/00031305.2016.1277159},
  keywords = {CAR score, CAT score, Cholesky decomposition, Decorrelation, Principal components analysis, Whitening, ZCA-Mahalanobis transformation},
  pages = {309--314},
  arxiv = {arXiv:1512.00809 [stat]},
  arxivdoi = {https://doi.org/10.48550/arXiv.1512.00809},
  tldr = {Covers `whitening', linear transforms that convert random vectors to another random vector, where the new random vector has covariance equal to the identity matrix. Five types discussed: zero-phase components analysis (ZCA) or Mahalanobis whitening, PCA whitening, Cholesky whitening, ZCA-cor, and PCA-cor. ZCA whitening is used in paper `CamP: Camera Preconditioning for Neural Radiance Fields', Park et al. 2023. `Whitening' is equivalent to the term `sphering'.}
}

Whitening, or sphering, is a common preprocessing step in statistical analysis to transform random variables to orthogonality. However, due to rotational freedom there are infinitely many possible whitening procedures. Consequently, there is a diverse range of sphering methods in use, for example, based on principal component analysis (PCA), Cholesky matrix decomposition, and zero-phase component analysis (ZCA), among others. Here, we provide an overview of the underlying theory and discuss five natural whitening procedures. Subsequently, we demonstrate that investigating the cross-covariance and the cross-correlation matrix between sphered and original variables allows to break the rotational invariance and to identify optimal whitening transformations. As a result we recommend two particular approaches: ZCA-cor whitening to produce sphered variables that are maximally similar to the original variables, and PCA-cor whitening to obtain sphered variables that maximally compress the original variables.

@article{park_camp_2023,
  author = {Park, Keunhong and Henzler, Philipp and Mildenhall, Ben and Barron, Jonathan T. and Martin-Brualla, Ricardo},
  title = {CamP: Camera Preconditioning for Neural Radiance Fields},
  journal = {ACM Trans. Graph.},
  publisher = {ACM},
  year = {2023},
  issue_date = {December 2023},
  freepdf = {https://arxiv.org/pdf/2308.10902},
  website = {https://camp-nerf.github.io/},
  arxiv = {arXiv:2308.10902 [cs.CV]},
  arxivdoi = {https://doi.org/10.48550/arXiv.2308.10902},
  tldr = {NeRF joint optimization of camera parameters and scene reconstruction. Uses a left preconditioner for each camera's parameters (Zero Component Analysis (ZCA) whitening transform (Kessy et al. 2018)), derived from a projection function; apply this at the initial iteration of the optimization. The new method is implemented on top of Zip-NeRF (Barron et al. 2023).}
}

Neural Radiance Fields (NeRF) can be optimized to obtain high-fidelity 3D scene reconstructions of objects and large-scale scenes. However, NeRFs require accurate camera parameters as input – inaccurate camera parameters result in blurry renderings. Extrinsic and intrinsic camera parameters are usually estimated using Structure-from-Motion (SfM) methods as a pre-processing step to NeRF, but these techniques rarely yield perfect estimates. Thus, prior works have proposed jointly optimizing camera parameters alongside a NeRF, but these methods are prone to local minima in challenging settings. In this work, we analyze how different camera parameterizations affect this joint optimization problem, and observe that standard parameterizations exhibit large differences in magnitude with respect to small perturbations, which can lead to an ill-conditioned optimization problem. We propose using a proxy problem to compute a whitening transform that eliminates the correlation between camera parameters and normalizes their effects, and we propose to use this transform as a preconditioner for the camera parameters during joint optimization. Our preconditioned camera optimization significantly improves reconstruction quality on scenes from the Mip-NeRF 360 dataset: we reduce error rates (RMSE) by 67% compared to state-of-the-art NeRF approaches that do not optimize for cameras like Zip-NeRF, and by 29% relative to state-of-the-art joint optimization approaches using the camera parameterization of SCNeRF. Our approach is easy to implement, does not significantly increase runtime, can be applied to a wide variety of camera parameterizations, and can straightforwardly be incorporated into other NeRF-like models.

@article{fleuret_little_2023,
  title = {The {Little} {Book} of {Deep} {Learning}},
  language = {en},
  author = {Fleuret, François},
  isbn = {9781447678618},
  year = {2023},
  freepdf = {https://fleuret.org/public/lbdl.pdf},
  website = {https://fleuret.org/francois/lbdl.html},
  tldr = {I really like this introduction to deep learning and reference guide. Want to remember a term without getting in too deep? This little book has it, and the top-level references if I want to read more. See the website to order a physical version, printing two book pages per printed page worked well for me too.}
}

This is a short introduction to deep learning for readers with a STEM background, originally designed to be read on a phone screen.

@incollection{rohrer_chapter6_2023,
  author = {Rohrer, Brandon},
  title = {Making Your Python Code Run Faster},
  chapter = {6},
  booktitle = {How to Train Your Robot},
  crossref = {rohrer_train_2023},
  year = {2023},
  urldate = {2023-08-30},
  url = {https://raw.githubusercontent.com/brohrer/how-to-train-your-robot/main/chapter_6/chapter_6.pdf},
  code = {https://github.com/brohrer/how-to-train-your-robot/tree/main/chapter_6},
  tldr = {Profiling, vectorization, pre-compilation with Numba, 10 optimization suggestions, "try it and test it", examples presented in context of a physics simulation. Good discussions about troubleshooting and debugging, when to visualize, determining project goals.},
  freepdf = {https://tyr.fyi/6},
  website = {https://tyr.fyi}
}

@article{kim_potential_2018,
  title = {Potential of {Unmanned} {Aerial} {Sampling} for {Monitoring} {Insect} {Populations} in {Rice} {Fields}},
  volume = {101},
  issn = {0015-4040, 1938-5102},
  url = {https://doi.org/10.1653/024.101.0229.full},
  doi = {10.1653/024.101.0229},
  number = {2},
  urldate = {2023-08-28},
  journal = {Florida Entomologist},
  author = {Kim, Hong Geun and Park, Jong-Seok and Lee, Doo-Hyung},
  month = jun,
  year = {2018},
  note = {Publisher: Florida Entomological Society},
  pages = {330--334},
  tldr = {Need to monitor for seasonal insect migrations in rice fields. Uses a UAS with small nets to collect samples of insects at different altitudes. To my knowledge, the only work to collect insects with a UAS versus using already-tagged insects.}
}

Conventionally, sampling for insects has been limited to the ground level or low altitudes. Recent progress in unmanned aerial vehicles has made it more feasible to use this technique for aerial sampling of insect populations. In this study, we developed a rotary-wing unmanned aerial vehicle with remote-controlled insect net openings that allows serial sampling at designated altitudes. A total of 21 flights using the unmanned aerial vehicle system captured 251 insects in 6 orders and 22 families at 5, 10, 50, and 100 m above rice fields in South Korea. The results of this study demonstrate that the aerial sampling can collect diverse pest and beneficial insects above rice fields and demonstrate a promising alternative to conventional sampling methods.

@misc{doersch_tutorial_2021,
  title = {Tutorial on {Variational} {Autoencoders}},
  url = {http://arxiv.org/abs/1606.05908},
  language = {en},
  urldate = {2023-08-28},
  publisher = {arXiv},
  author = {Doersch, Carl},
  month = jan,
  year = {2021},
  arxiv = {arXiv:1606.05908 [cs, stat]},
  arxivdoi = {https://doi.org/10.48550/arXiv.1606.05908},
  keywords = {Computer Science - Machine Learning, Statistics - Machine Learning},
  code = {https://github.com/cdoersch/vae_tutorial},
  tldr = {Published 2016 with edits in 2021. Tutorial on Variational Autoencoders, reparameterization trick, and Conditional Variational Autoencoders. Examples using MNIST. Does not assume prior knowledge of  variational Bayesian methods.}
}

In just three years, Variational Autoencoders (VAEs) have emerged as one of the most popular approaches to unsupervised learning of complicated distributions. VAEs are appealing because they are built on top of standard function approximators (neural networks), and can be trained with stochastic gradient descent. VAEs have already shown promise in generating many kinds of complicated data, including handwritten digits [1, 2], faces [1, 3, 4], house numbers [5, 6], CIFAR images [6], physical models of scenes [4], segmentation [7], and predicting the future from static images [8]. This tutorial introduces the intuitions behind VAEs, explains the mathematics behind them, and describes some empirical behavior. No prior knowledge of variational Bayesian methods is assumed.