WarpX.H

Go to the documentation of this file.

/* Copyright 2016-2020 Andrew Myers, Ann Almgren, Aurore Blelly
 *                     Axel Huebl, Burlen Loring, David Grote
 *                     Glenn Richardson, Junmin Gu, Luca Fedeli
 *                     Mathieu Lobet, Maxence Thevenet, Michael Rowan
 *                     Remi Lehe, Revathi Jambunathan, Weiqun Zhang
 *                     Yinjian Zhao
 *
 * This file is part of WarpX.
 *
 * License: BSD-3-Clause-LBNL
 */
#ifndef WARPX_H_
#define WARPX_H_
 
#include "BoundaryConditions/PML_fwd.H"
#include "Diagnostics/MultiDiagnostics_fwd.H"
#include "Diagnostics/ReducedDiags/MultiReducedDiags_fwd.H"
#include "EmbeddedBoundary/WarpXFaceInfoBox_fwd.H"
#include "FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver_fwd.H"
#include "FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties_fwd.H"
#include "FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel_fwd.H"
#include "Filter/NCIGodfreyFilter_fwd.H"
#include "Particles/ParticleBoundaryBuffer_fwd.H"
#include "Particles/MultiParticleContainer_fwd.H"
#include "Particles/WarpXParticleContainer_fwd.H"
#ifdef WARPX_USE_PSATD
#   ifdef WARPX_DIM_RZ
#       include "FieldSolver/SpectralSolver/SpectralSolverRZ_fwd.H"
#       include "BoundaryConditions/PML_RZ_fwd.H"
#   else
#       include "FieldSolver/SpectralSolver/SpectralSolver_fwd.H"
#   endif
#endif
#include "Evolve/WarpXDtType.H"
#include "FieldSolver/ElectrostaticSolver.H"
#include "FieldSolver/MagnetostaticSolver/MagnetostaticSolver.H"
#include "Filter/BilinearFilter.H"
#include "Parallelization/GuardCellManager.H"
#include "AcceleratorLattice/AcceleratorLattice.H"
#include "Utils/Parser/IntervalsParser.H"
#include "Utils/WarpXAlgorithmSelection.H"
 
#include <AMReX.H>
#include <AMReX_AmrCore.H>
#include <AMReX_Array.H>
#include <AMReX_Config.H>
#ifdef AMREX_USE_EB
#   include <AMReX_EBFabFactory.H>
#endif
#include <AMReX_GpuContainers.H>
#include <AMReX_IntVect.H>
#include <AMReX_LayoutData.H>
#include <AMReX_Parser.H>
#include <AMReX_REAL.H>
#include <AMReX_RealBox.H>
#include <AMReX_RealVect.H>
#include <AMReX_Vector.H>
#include <AMReX_VisMF.H>
 
#include <AMReX_BaseFwd.H>
#include <AMReX_AmrCoreFwd.H>
 
#include <array>
#include <iostream>
#include <limits>
#include <memory>
#include <optional>
#include <string>
#include <vector>
#include <map>
 
enum struct PatchType : int
{
    fine,
    coarse
};
 
class WarpX
    : public amrex::AmrCore
{
public:
 
    friend class PML;
 
    static WarpX& GetInstance ();
 
    static void ResetInstance ();
 
    static void Finalize();
 
    ~WarpX ();
 
    WarpX ( WarpX const &) = delete;
    WarpX& operator= ( WarpX const & ) = delete;
 
    WarpX ( WarpX && ) = default;
    WarpX& operator= ( WarpX && ) = default;
 
    static std::string Version (); 
    static std::string PicsarVersion (); 
 
    int Verbose () const { return verbose; }
 
    void InitData ();
 
    void Evolve (int numsteps = -1);
 
    MultiParticleContainer& GetPartContainer () { return *mypc; }
    MacroscopicProperties& GetMacroscopicProperties () { return *m_macroscopic_properties; }
    HybridPICModel& GetHybridPICModel () { return *m_hybrid_pic_model; }
    MultiDiagnostics& GetMultiDiags () {return *multi_diags;}
 
    ParticleBoundaryBuffer& GetParticleBoundaryBuffer () { return *m_particle_boundary_buffer; }
 
    static void shiftMF (amrex::MultiFab& mf, const amrex::Geometry& geom,
                         int num_shift, int dir, const int lev, bool update_cost_flag,
                         amrex::Real external_field=0.0, bool useparser = false,
                         amrex::ParserExecutor<3> const& field_parser={});
 
    static std::string authors;
 
    static amrex::Vector<amrex::Real> E_external_grid;
    static amrex::Vector<amrex::Real> B_external_grid;
 
    static std::string B_ext_grid_s;
    static std::string E_ext_grid_s;
 
    static bool add_external_E_field;
    static bool add_external_B_field;
 
    static std::string str_Bx_ext_grid_function;
    static std::string str_By_ext_grid_function;
    static std::string str_Bz_ext_grid_function;
    static std::string str_Ex_ext_grid_function;
    static std::string str_Ey_ext_grid_function;
    static std::string str_Ez_ext_grid_function;
 
    std::unique_ptr<amrex::Parser> Bxfield_parser;
    std::unique_ptr<amrex::Parser> Byfield_parser;
    std::unique_ptr<amrex::Parser> Bzfield_parser;
    std::unique_ptr<amrex::Parser> Exfield_parser;
    std::unique_ptr<amrex::Parser> Eyfield_parser;
    std::unique_ptr<amrex::Parser> Ezfield_parser;
 
    // Algorithms
    static short current_deposition_algo;
    static short charge_deposition_algo;
    static short field_gathering_algo;
    static short particle_pusher_algo;
    static short electromagnetic_solver_id;
    static short load_balance_costs_update_algo;
    static int em_solver_medium;
    static int macroscopic_solver_algo;
    static amrex::Vector<int> field_boundary_lo;
    static amrex::Vector<int> field_boundary_hi;
    static amrex::Vector<ParticleBoundaryType> particle_boundary_lo;
    static amrex::Vector<ParticleBoundaryType> particle_boundary_hi;
 
    static short psatd_solution_type;
 
    static short J_in_time;
    static short rho_in_time;
 
    static bool do_current_centering;
 
    //  to satisfy the continuity equation and charge conservation
    bool current_correction;
 
    bool update_with_rho = false;
 
    static bool do_single_precision_comms;
 
    static bool do_shared_mem_charge_deposition;
 
    static bool do_shared_mem_current_deposition;
 
    static int shared_mem_current_tpb;
 
    static amrex::IntVect shared_tilesize;
 
    static amrex::IntVect m_fill_guards_fields;
 
    static amrex::IntVect m_fill_guards_current;
 
    static bool do_dive_cleaning;
    static bool do_divb_cleaning;
 
    static int nox;
    static int noy;
    static int noz;
 
    static int field_centering_nox;
    static int field_centering_noy;
    static int field_centering_noz;
 
    static int current_centering_nox;
    static int current_centering_noy;
    static int current_centering_noz;
 
    static int n_rz_azimuthal_modes;
    static int ncomps;
 
    static bool use_fdtd_nci_corr;
    static bool galerkin_interpolation;
 
    static bool verboncoeur_axis_correction;
 
    static bool use_filter;
    static bool use_kspace_filter;
    static bool use_filter_compensation;
 
    static bool serialize_initial_conditions;
 
    static amrex::Real gamma_boost;
    static amrex::Real beta_boost;
    static amrex::Vector<int> boost_direction;
    static amrex::Real zmax_plasma_to_compute_max_step;
    static bool do_compute_max_step_from_zmax;
    static amrex::Real zmin_domain_boost_step_0;
 
    static bool compute_max_step_from_btd;
 
    static bool do_dynamic_scheduling;
    static bool refine_plasma;
 
    static utils::parser::IntervalsParser sort_intervals;
    static amrex::IntVect sort_bin_size;
 
    static bool sort_particles_for_deposition;
    static amrex::IntVect sort_idx_type;
 
    static bool do_subcycling;
    static bool do_multi_J;
    static int do_multi_J_n_depositions;
 
    static bool do_device_synchronize;
    static bool safe_guard_cells;
 
    static int n_field_gather_buffer;
    static int n_current_deposition_buffer;
 
    static short grid_type;
 
    // Global rho nodal flag to know about rho index type when rho MultiFab is not allocated
    amrex::IntVect m_rho_nodal_flag;
 
    static void AllocInitMultiFab (
        std::unique_ptr<amrex::MultiFab>& mf,
        const amrex::BoxArray& ba,
        const amrex::DistributionMapping& dm,
        const int ncomp,
        const amrex::IntVect& ngrow,
        const int level,
        const std::string& name,
        std::optional<const amrex::Real> initial_value = {});
 
    static void AllocInitMultiFab (
        std::unique_ptr<amrex::iMultiFab>& mf,
        const amrex::BoxArray& ba,
        const amrex::DistributionMapping& dm,
        const int ncomp,
        const amrex::IntVect& ngrow,
        const int level,
        const std::string& name,
        std::optional<const int> initial_value = {});
 
    static void AliasInitMultiFab (
        std::unique_ptr<amrex::MultiFab>& mf,
        const amrex::MultiFab& mf_to_alias,
        const int scomp,
        const int ncomp,
        const int level,
        const std::string& name,
        std::optional<const amrex::Real> initial_value);
 
    // Maps of all of the MultiFabs and iMultiFabs used (this can include MFs from other classes)
    // This is a convenience for the Python interface, allowing all MultiFabs
    // to be easily referenced from Python.
    static std::map<std::string, amrex::MultiFab *> multifab_map;
    static std::map<std::string, amrex::iMultiFab *> imultifab_map;
 
    static void AddToMultiFabMap(const std::string& name, const std::unique_ptr<amrex::MultiFab>& mf) {
        multifab_map[name] = mf.get();
    }
 
    static void AddToMultiFabMap(const std::string& name, const std::unique_ptr<amrex::iMultiFab>& mf) {
        imultifab_map[name] = mf.get();
    }
 
    std::array<const amrex::MultiFab* const, 3>
    get_array_Bfield_aux  (const int lev) const {
        return {
            Bfield_aux[lev][0].get(),
            Bfield_aux[lev][1].get(),
            Bfield_aux[lev][2].get()
        };
    }
    std::array<const amrex::MultiFab* const, 3>
    get_array_Efield_aux  (const int lev) const {
        return {
            Efield_aux[lev][0].get(),
            Efield_aux[lev][1].get(),
            Efield_aux[lev][2].get()
        };
    }
 
    amrex::MultiFab * get_pointer_Efield_aux  (int lev, int direction) const { return Efield_aux[lev][direction].get(); }
    amrex::MultiFab * get_pointer_Bfield_aux  (int lev, int direction) const { return Bfield_aux[lev][direction].get(); }
 
    amrex::MultiFab * get_pointer_Efield_fp  (int lev, int direction) const { return Efield_fp[lev][direction].get(); }
    amrex::MultiFab * get_pointer_Bfield_fp  (int lev, int direction) const { return Bfield_fp[lev][direction].get(); }
    amrex::MultiFab * get_pointer_current_fp  (int lev, int direction) const { return current_fp[lev][direction].get(); }
    amrex::MultiFab * get_pointer_current_fp_nodal  (int lev, int direction) const { return current_fp_nodal[lev][direction].get(); }
    amrex::MultiFab * get_pointer_rho_fp  (int lev) const { return rho_fp[lev].get(); }
    amrex::MultiFab * get_pointer_F_fp  (int lev) const { return F_fp[lev].get(); }
    amrex::MultiFab * get_pointer_G_fp  (int lev) const { return G_fp[lev].get(); }
    amrex::MultiFab * get_pointer_phi_fp  (int lev) const { return phi_fp[lev].get(); }
    amrex::MultiFab * get_pointer_vector_potential_fp  (int lev, int direction) const { return vector_potential_fp_nodal[lev][direction].get(); }
 
    amrex::MultiFab * get_pointer_Efield_cp  (int lev, int direction) const { return Efield_cp[lev][direction].get(); }
    amrex::MultiFab * get_pointer_Bfield_cp  (int lev, int direction) const { return Bfield_cp[lev][direction].get(); }
    amrex::MultiFab * get_pointer_current_cp  (int lev, int direction) const { return current_cp[lev][direction].get(); }
    amrex::MultiFab * get_pointer_rho_cp  (int lev) const { return rho_cp[lev].get(); }
    amrex::MultiFab * get_pointer_F_cp  (int lev) const { return F_cp[lev].get(); }
    amrex::MultiFab * get_pointer_G_cp  (int lev) const { return G_cp[lev].get(); }
 
    amrex::MultiFab * get_pointer_edge_lengths  (int lev, int direction) const { return m_edge_lengths[lev][direction].get(); }
    amrex::MultiFab * get_pointer_face_areas  (int lev, int direction) const { return m_face_areas[lev][direction].get(); }
 
    const amrex::MultiFab& getEfield  (int lev, int direction) {return *Efield_aux[lev][direction];}
    const amrex::MultiFab& getBfield  (int lev, int direction) {return *Bfield_aux[lev][direction];}
 
    const amrex::MultiFab& getcurrent_cp (int lev, int direction) {return *current_cp[lev][direction];}
    const amrex::MultiFab& getEfield_cp  (int lev, int direction) {return  *Efield_cp[lev][direction];}
    const amrex::MultiFab& getBfield_cp  (int lev, int direction) {return  *Bfield_cp[lev][direction];}
    const amrex::MultiFab& getrho_cp (int lev) {return  *rho_cp[lev];}
    const amrex::MultiFab& getF_cp (int lev) {return *F_cp[lev];}
    const amrex::MultiFab& getG_cp (int lev) {return *G_cp[lev];}
 
    const amrex::MultiFab& getcurrent_fp (int lev, int direction) {return *current_fp[lev][direction];}
    const amrex::MultiFab& getEfield_fp  (int lev, int direction) {return *Efield_fp[lev][direction];}
    const amrex::MultiFab& getBfield_fp  (int lev, int direction) {return *Bfield_fp[lev][direction];}
    const amrex::MultiFab& getrho_fp (int lev) {return *rho_fp[lev];}
    const amrex::MultiFab& getphi_fp (int lev) {return *phi_fp[lev];}
    const amrex::MultiFab& getF_fp (int lev) {return *F_fp[lev];}
    const amrex::MultiFab& getG_fp (int lev) {return *G_fp[lev];}
 
    const amrex::MultiFab& getEfield_avg_fp (int lev, int direction) {return *Efield_avg_fp[lev][direction];}
    const amrex::MultiFab& getBfield_avg_fp (int lev, int direction) {return *Bfield_avg_fp[lev][direction];}
    const amrex::MultiFab& getEfield_avg_cp (int lev, int direction) {return *Efield_avg_cp[lev][direction];}
    const amrex::MultiFab& getBfield_avg_cp (int lev, int direction) {return *Bfield_avg_cp[lev][direction];}
 
    bool DoPML () const {return do_pml;}
 
#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD)
    const PML_RZ* getPMLRZ() {return pml_rz[0].get();}
#endif
 
    std::vector<bool> getPMLdirections() const;
 
    static amrex::LayoutData<amrex::Real>* getCosts (int lev);
 
    void setLoadBalanceEfficiency (const int lev, const amrex::Real efficiency);
 
    amrex::Real getLoadBalanceEfficiency (const int lev);
 
    static amrex::IntVect filter_npass_each_dir;
    BilinearFilter bilinear_filter;
    amrex::Vector< std::unique_ptr<NCIGodfreyFilter> > nci_godfrey_filter_exeybz;
    amrex::Vector< std::unique_ptr<NCIGodfreyFilter> > nci_godfrey_filter_bxbyez;
 
    amrex::Real time_of_last_gal_shift = 0;
    amrex::Vector<amrex::Real> m_v_galilean = amrex::Vector<amrex::Real>(3, amrex::Real(0.));
    amrex::Array<amrex::Real,3> m_galilean_shift = {{0}};
 
    amrex::Vector<amrex::Real> m_v_comoving = amrex::Vector<amrex::Real>(3, amrex::Real(0.));
 
    static int num_mirrors;
    amrex::Vector<amrex::Real> mirror_z;
    amrex::Vector<amrex::Real> mirror_z_width;
    amrex::Vector<int> mirror_z_npoints;
 
    std::unique_ptr<MultiReducedDiags> reduced_diags;
 
    void applyMirrors(amrex::Real time);
 
    void ComputeDt ();
 
    void PrintMainPICparameters ();
 
    void WriteUsedInputsFile () const;
 
    void PrintDtDxDyDz ();
 
    void ComputeMaxStep ();
    // Compute max_step automatically for simulations in a boosted frame.
    void computeMaxStepBoostAccelerator();
 
    int  MoveWindow (const int step, bool move_j);
 
    void ShiftGalileanBoundary ();
    void UpdatePlasmaInjectionPosition (amrex::Real dt);
    void ResetProbDomain (const amrex::RealBox& rb);
    void EvolveE (         amrex::Real dt);
    void EvolveE (int lev, amrex::Real dt);
    void EvolveB (         amrex::Real dt, DtType dt_type);
    void EvolveB (int lev, amrex::Real dt, DtType dt_type);
    void EvolveF (         amrex::Real dt, DtType dt_type);
    void EvolveF (int lev, amrex::Real dt, DtType dt_type);
    void EvolveG (         amrex::Real dt, DtType dt_type);
    void EvolveG (int lev, amrex::Real dt, DtType dt_type);
    void EvolveB (int lev, PatchType patch_type, amrex::Real dt, DtType dt_type);
    void EvolveE (int lev, PatchType patch_type, amrex::Real dt);
    void EvolveF (int lev, PatchType patch_type, amrex::Real dt, DtType dt_type);
    void EvolveG (int lev, PatchType patch_type, amrex::Real dt, DtType dt_type);
 
    void MacroscopicEvolveE (         amrex::Real dt);
    void MacroscopicEvolveE (int lev, amrex::Real dt);
    void MacroscopicEvolveE (int lev, PatchType patch_type, amrex::Real dt);
 
    void HybridPICEvolveFields ();
 
    void HybridPICDepositInitialRhoAndJ ();
 
    void Hybrid_QED_Push (         amrex::Vector<amrex::Real> dt);
 
    void Hybrid_QED_Push (int lev, amrex::Real dt);
 
    void Hybrid_QED_Push (int lev, PatchType patch_type, amrex::Real dt);
 
    static amrex::Real quantum_xi_c2;
 
    void LoadBalance ();
    void ResetCosts ();
 
    utils::parser::IntervalsParser get_load_balance_intervals () const
    {
        return load_balance_intervals;
    }
 
    void DampFieldsInGuards (const int lev,
                             const std::array<std::unique_ptr<amrex::MultiFab>,3>& Efield,
                             const std::array<std::unique_ptr<amrex::MultiFab>,3>& Bfield);
 
    void DampFieldsInGuards (const int lev, std::unique_ptr<amrex::MultiFab>& mf);
 
#ifdef WARPX_DIM_RZ
    void ApplyInverseVolumeScalingToCurrentDensity(amrex::MultiFab* Jx,
                                                   amrex::MultiFab* Jy,
                                                   amrex::MultiFab* Jz,
                                                   int lev);
 
    void ApplyInverseVolumeScalingToChargeDensity(amrex::MultiFab* Rho,
                                                  int lev);
#endif
 
    void ApplyRhofieldBoundary (const int lev, amrex::MultiFab* Rho,
                                PatchType patch_type);
 
    void ApplyJfieldBoundary (const int lev, amrex::MultiFab* Jx,
                              amrex::MultiFab* Jy, amrex::MultiFab* Jz,
                              PatchType patch_type);
 
    void ApplyEfieldBoundary (const int lev, PatchType patch_type);
    void ApplyBfieldBoundary (const int lev, PatchType patch_type, DtType dt_type);
 
    void ApplyElectronPressureBoundary (const int lev, PatchType patch_type);
 
    void DampPML ();
    void DampPML (const int lev);
    void DampPML (const int lev, PatchType patch_type);
    void DampPML_Cartesian (const int lev, PatchType patch_type);
 
    void DampJPML ();
    void DampJPML (int lev);
    void DampJPML (int lev, PatchType patch_type);
 
    void CopyJPML ();
    bool isAnyBoundaryPML();
 
    void NodalSyncPML ();
 
    void NodalSyncPML (int lev);
 
    void NodalSyncPML (int lev, PatchType patch_type);
 
    PML* GetPML (int lev);
#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD)
    PML_RZ* GetPML_RZ (int lev);
#endif
 
    void doFieldIonization ();
    void doFieldIonization (int lev);
 
#ifdef WARPX_QED
    void doQEDEvents ();
    void doQEDEvents (int lev);
#endif
 
    void PushParticlesandDepose (int lev, amrex::Real cur_time, DtType a_dt_type=DtType::Full, bool skip_current=false);
    void PushParticlesandDepose (         amrex::Real cur_time, bool skip_current=false);
 
    // This function does aux(lev) = fp(lev) + I(aux(lev-1)-cp(lev)).
    // Caller must make sure fp and cp have ghost cells filled.
    void UpdateAuxilaryData ();
    void UpdateAuxilaryDataStagToNodal ();
    void UpdateAuxilaryDataSameType ();
 
    void UpdateCurrentNodalToStag (amrex::MultiFab& dst, amrex::MultiFab const& src);
 
    // Fill boundary cells including coarse/fine boundaries
    void FillBoundaryB   (amrex::IntVect ng, std::optional<bool> nodal_sync = std::nullopt);
    void FillBoundaryE   (amrex::IntVect ng, std::optional<bool> nodal_sync = std::nullopt);
    void FillBoundaryB_avg   (amrex::IntVect ng);
    void FillBoundaryE_avg   (amrex::IntVect ng);
 
    void FillBoundaryF   (amrex::IntVect ng, std::optional<bool> nodal_sync = std::nullopt);
    void FillBoundaryG   (amrex::IntVect ng, std::optional<bool> nodal_sync = std::nullopt);
    void FillBoundaryAux (amrex::IntVect ng);
    void FillBoundaryE   (int lev, amrex::IntVect ng, std::optional<bool> nodal_sync = std::nullopt);
    void FillBoundaryB   (int lev, amrex::IntVect ng, std::optional<bool> nodal_sync = std::nullopt);
    void FillBoundaryE_avg   (int lev, amrex::IntVect ng);
    void FillBoundaryB_avg   (int lev, amrex::IntVect ng);
 
    void FillBoundaryF   (int lev, amrex::IntVect ng, std::optional<bool> nodal_sync = std::nullopt);
    void FillBoundaryG   (int lev, amrex::IntVect ng, std::optional<bool> nodal_sync = std::nullopt);
    void FillBoundaryAux (int lev, amrex::IntVect ng);
 
    void SyncCurrentAndRho ();
 
    void SyncCurrent (
        const amrex::Vector<std::array<std::unique_ptr<amrex::MultiFab>,3>>& J_fp,
        const amrex::Vector<std::array<std::unique_ptr<amrex::MultiFab>,3>>& J_cp,
        const amrex::Vector<std::array<std::unique_ptr<amrex::MultiFab>,3>>& J_buffer);
 
    void SyncRho ();
 
    void SyncRho (
        const amrex::Vector<std::unique_ptr<amrex::MultiFab>>& charge_fp,
        const amrex::Vector<std::unique_ptr<amrex::MultiFab>>& charge_cp,
        const amrex::Vector<std::unique_ptr<amrex::MultiFab>>& charge_buffer);
 
    amrex::Vector<int> getnsubsteps () const {return nsubsteps;}
    int getnsubsteps (int lev) const {return nsubsteps[lev];}
    amrex::Vector<int> getistep () const {return istep;}
    int getistep (int lev) const {return istep[lev];}
    void setistep (int lev, int ii) {istep[lev] = ii;}
    amrex::Vector<amrex::Real> gett_old () const {return t_old;}
    amrex::Real gett_old (int lev) const {return t_old[lev];}
    amrex::Vector<amrex::Real> gett_new () const {return t_new;}
    amrex::Real gett_new (int lev) const {return t_new[lev];}
    void sett_new (int lev, amrex::Real time) {t_new[lev] = time;}
    amrex::Vector<amrex::Real> getdt () const {return dt;}
    amrex::Real getdt (int lev) const {return dt[lev];}
    int getdo_moving_window() const {return do_moving_window;}
    amrex::Real getmoving_window_x() const {return moving_window_x;}
    bool getis_synchronized() const {return is_synchronized;}
 
    int maxStep () const {return max_step;}
    void updateMaxStep (const int new_max_step) {max_step = new_max_step;}
    amrex::Real stopTime () const {return stop_time;}
    void updateStopTime (const amrex::Real new_stop_time) {stop_time = new_stop_time;}
 
    void AverageAndPackFields( amrex::Vector<std::string>& varnames,
        amrex::Vector<amrex::MultiFab>& mf_avg, const amrex::IntVect ngrow) const;
 
    void prepareFields( int const step, amrex::Vector<std::string>& varnames,
        amrex::Vector<amrex::MultiFab>& mf_avg,
        amrex::Vector<const amrex::MultiFab*>& output_mf,
        amrex::Vector<amrex::Geometry>& output_geom ) const;
 
    static std::array<amrex::Real,3> CellSize (int lev);
    static amrex::RealBox getRealBox(const amrex::Box& bx, int lev);
 
    static std::array<amrex::Real,3> LowerCorner (const amrex::Box& bx, const int lev, const amrex::Real time_shift_delta);
    static std::array<amrex::Real,3> UpperCorner (const amrex::Box& bx, const int lev, const amrex::Real time_shift_delta);
 
    static amrex::IntVect RefRatio (int lev);
 
    static const amrex::iMultiFab* CurrentBufferMasks (int lev);
    static const amrex::iMultiFab* GatherBufferMasks (int lev);
 
    static int electrostatic_solver_id;
 
    // Parameters for lab frame electrostatic
    static amrex::Real self_fields_required_precision;
    static amrex::Real self_fields_absolute_tolerance;
    static int self_fields_max_iters;
    static int self_fields_verbosity;
 
    static int do_moving_window; // boolean
    static int start_moving_window_step; // the first step to move window
    static int end_moving_window_step; // the last step to move window
    static int moving_window_active (int const step) {
        bool const step_before_end = (step < end_moving_window_step) || (end_moving_window_step < 0);
        bool const step_after_start = (step >= start_moving_window_step);
        return do_moving_window && step_before_end && step_after_start;
    }
    static int moving_window_dir;
    static amrex::Real moving_window_v;
    static bool fft_do_time_averaging;
 
    // these should be private, but can't due to Cuda limitations
    static void ComputeDivB (amrex::MultiFab& divB, int const dcomp,
                             const std::array<const amrex::MultiFab* const, 3>& B,
                             const std::array<amrex::Real,3>& dx);
 
    static void ComputeDivB (amrex::MultiFab& divB, int const dcomp,
                             const std::array<const amrex::MultiFab* const, 3>& B,
                             const std::array<amrex::Real,3>& dx, amrex::IntVect const ngrow);
 
    void ComputeDivE(amrex::MultiFab& divE, const int lev);
 
    const amrex::IntVect getngEB() const { return guard_cells.ng_alloc_EB; }
    const amrex::IntVect getngF() const { return guard_cells.ng_alloc_F; }
    const amrex::IntVect getngUpdateAux() const { return guard_cells.ng_UpdateAux; }
    const amrex::IntVect get_ng_depos_J() const {return guard_cells.ng_depos_J;}
    const amrex::IntVect get_ng_depos_rho() const {return guard_cells.ng_depos_rho;}
    const amrex::IntVect get_ng_fieldgather () const {return guard_cells.ng_FieldGather;}
 
    const amrex::IntVect get_numprocs() const {return numprocs;}
 
    ElectrostaticSolver::PoissonBoundaryHandler m_poisson_boundary_handler;
    void ComputeSpaceChargeField (bool const reset_fields);
    void AddBoundaryField ();
    void AddSpaceChargeField (WarpXParticleContainer& pc);
    void AddSpaceChargeFieldLabFrame ();
    void computePhi (const amrex::Vector<std::unique_ptr<amrex::MultiFab> >& rho,
                     amrex::Vector<std::unique_ptr<amrex::MultiFab> >& phi,
                     std::array<amrex::Real, 3> const beta = {{0,0,0}},
                     amrex::Real const required_precision=amrex::Real(1.e-11),
                     amrex::Real absolute_tolerance=amrex::Real(0.0),
                     const int max_iters=200,
                     const int verbosity=2) const;
 
    void setPhiBC (amrex::Vector<std::unique_ptr<amrex::MultiFab> >& phi ) const;
 
    void computeE (amrex::Vector<std::array<std::unique_ptr<amrex::MultiFab>, 3> >& E,
                   const amrex::Vector<std::unique_ptr<amrex::MultiFab> >& phi,
                   std::array<amrex::Real, 3> const beta = {{0,0,0}} ) const;
    void computeB (amrex::Vector<std::array<std::unique_ptr<amrex::MultiFab>, 3> >& B,
                   const amrex::Vector<std::unique_ptr<amrex::MultiFab> >& phi,
                   std::array<amrex::Real, 3> const beta = {{0,0,0}} ) const;
    void computePhiTriDiagonal (const amrex::Vector<std::unique_ptr<amrex::MultiFab> >& rho,
                                      amrex::Vector<std::unique_ptr<amrex::MultiFab> >& phi) const;
 
    // Magnetostatic Solver Interface
    MagnetostaticSolver::VectorPoissonBoundaryHandler m_vector_poisson_boundary_handler;
    void ComputeMagnetostaticField ();
    void AddMagnetostaticFieldLabFrame ();
    void computeVectorPotential (const amrex::Vector<amrex::Array<std::unique_ptr<amrex::MultiFab>, 3> >& curr,
                                 amrex::Vector<amrex::Array<std::unique_ptr<amrex::MultiFab>, 3> >& A,
                                 amrex::Real const required_precision=amrex::Real(1.e-11),
                                 amrex::Real absolute_tolerance=amrex::Real(0.0),
                                 const int max_iters=200,
                                 const int verbosity=2) const;
 
    void setVectorPotentialBC (amrex::Vector<amrex::Array<std::unique_ptr<amrex::MultiFab>, 3> >& A) const;
 
    void InitializeExternalFieldsOnGridUsingParser (
         amrex::MultiFab *mfx, amrex::MultiFab *mfy, amrex::MultiFab *mfz,
         amrex::ParserExecutor<3> const& xfield_parser,
         amrex::ParserExecutor<3> const& yfield_parser,
         amrex::ParserExecutor<3> const& zfield_parser,
         std::array< std::unique_ptr<amrex::MultiFab>, 3 > const& edge_lengths,
         std::array< std::unique_ptr<amrex::MultiFab>, 3 > const& face_areas,
         const char field,
         const int lev, PatchType patch_type);
 
    void ReadExternalFieldFromFile (
         std::string read_fields_from_path, amrex::MultiFab* mf,
         std::string F_name, std::string F_component);
 
    void InitializeEBGridData(int lev);
 
    void ComputeCostsHeuristic (amrex::Vector<std::unique_ptr<amrex::LayoutData<amrex::Real> > >& costs);
 
    void ApplyFilterandSumBoundaryRho (int lev, int glev, amrex::MultiFab& rho, int icomp, int ncomp);
 
    static amrex::Vector<amrex::Real> getFornbergStencilCoefficients(const int n_order, const short a_grid_type);
 
    // Device vectors of stencil coefficients used for finite-order centering of fields
    amrex::Gpu::DeviceVector<amrex::Real> device_field_centering_stencil_coeffs_x;
    amrex::Gpu::DeviceVector<amrex::Real> device_field_centering_stencil_coeffs_y;
    amrex::Gpu::DeviceVector<amrex::Real> device_field_centering_stencil_coeffs_z;
 
    // Device vectors of stencil coefficients used for finite-order centering of currents
    amrex::Gpu::DeviceVector<amrex::Real> device_current_centering_stencil_coeffs_x;
    amrex::Gpu::DeviceVector<amrex::Real> device_current_centering_stencil_coeffs_y;
    amrex::Gpu::DeviceVector<amrex::Real> device_current_centering_stencil_coeffs_z;
 
    // This needs to be public for CUDA.
    virtual void ErrorEst (int lev, amrex::TagBoxArray& tags, amrex::Real time, int /*ngrow*/) final;
 
    // Return the accelerator lattice instance defined at the given refinement level
    const AcceleratorLattice& get_accelerator_lattice (int lev) {return *(m_accelerator_lattice[lev]);}
 
protected:
 
    void InitLevelData (int lev, amrex::Real time);
 
    virtual void PostProcessBaseGrids (amrex::BoxArray& ba0) const final;
 
    virtual void MakeNewLevelFromScratch (int lev, amrex::Real time, const amrex::BoxArray& ba,
                                          const amrex::DistributionMapping& dm) final;
 
    virtual void MakeNewLevelFromCoarse (int /*lev*/, amrex::Real /*time*/, const amrex::BoxArray& /*ba*/,
                                         const amrex::DistributionMapping& /*dm*/) final;
 
    virtual void RemakeLevel (int lev, amrex::Real time, const amrex::BoxArray& ba,
                              const amrex::DistributionMapping& dm) final;
 
    virtual void ClearLevel (int lev) final;
 
private:
 
    WarpX ();
 
    static void MakeWarpX ();
 
    // Singleton is used when the code is run from python
    static WarpX* m_instance;
 
    static void CheckSignals ();
    void HandleSignals ();
 
    void EvolveEM(int numsteps);
 
    void FillBoundaryB (const int lev, const PatchType patch_type, const amrex::IntVect ng, std::optional<bool> nodal_sync = std::nullopt);
    void FillBoundaryE (const int lev, const PatchType patch_type, const amrex::IntVect ng, std::optional<bool> nodal_sync = std::nullopt);
    void FillBoundaryF (int lev, PatchType patch_type, amrex::IntVect ng, std::optional<bool> nodal_sync = std::nullopt);
    void FillBoundaryG (int lev, PatchType patch_type, amrex::IntVect ng, std::optional<bool> nodal_sync = std::nullopt);
 
    void FillBoundaryB_avg (int lev, PatchType patch_type, amrex::IntVect ng);
    void FillBoundaryE_avg (int lev, PatchType patch_type, amrex::IntVect ng);
 
    void AddExternalFields ();
 
    void OneStep_nosub (amrex::Real t);
    void OneStep_sub1 (amrex::Real t);
 
    void OneStep_multiJ (const amrex::Real t);
 
    void RestrictCurrentFromFineToCoarsePatch (
        const amrex::Vector<std::array<std::unique_ptr<amrex::MultiFab>,3>>& J_fp,
        const amrex::Vector<std::array<std::unique_ptr<amrex::MultiFab>,3>>& J_cp,
        const int lev);
    void AddCurrentFromFineLevelandSumBoundary (
        const amrex::Vector<std::array<std::unique_ptr<amrex::MultiFab>,3>>& J_fp,
        const amrex::Vector<std::array<std::unique_ptr<amrex::MultiFab>,3>>& J_cp,
        const amrex::Vector<std::array<std::unique_ptr<amrex::MultiFab>,3>>& J_buffer,
        const int lev);
    void StoreCurrent (const int lev);
    void RestoreCurrent (const int lev);
    void ApplyFilterJ (
        const amrex::Vector<std::array<std::unique_ptr<amrex::MultiFab>,3>>& current,
        const int lev,
        const int idim);
    void ApplyFilterJ (
        const amrex::Vector<std::array<std::unique_ptr<amrex::MultiFab>,3>>& current,
        const int lev);
    void SumBoundaryJ (
        const amrex::Vector<std::array<std::unique_ptr<amrex::MultiFab>,3>>& current,
        const int lev,
        const int idim,
        const amrex::Periodicity& period);
    void SumBoundaryJ (
        const amrex::Vector<std::array<std::unique_ptr<amrex::MultiFab>,3>>& current,
        const int lev,
        const amrex::Periodicity& period);
    void NodalSyncJ (
        const amrex::Vector<std::array<std::unique_ptr<amrex::MultiFab>,3>>& J_fp,
        const amrex::Vector<std::array<std::unique_ptr<amrex::MultiFab>,3>>& J_cp,
        const int lev,
        PatchType patch_type);
 
    void RestrictRhoFromFineToCoarsePatch (
        const amrex::Vector<std::unique_ptr<amrex::MultiFab>>& charge_fp,
        const amrex::Vector<std::unique_ptr<amrex::MultiFab>>& charge_cp,
        const int lev);
    void ApplyFilterandSumBoundaryRho (
        const amrex::Vector<std::unique_ptr<amrex::MultiFab>>& charge_fp,
        const amrex::Vector<std::unique_ptr<amrex::MultiFab>>& charge_cp,
        const int lev,
        PatchType patch_type,
        const int icomp,
        const int ncomp);
    void AddRhoFromFineLevelandSumBoundary (
        const amrex::Vector<std::unique_ptr<amrex::MultiFab>>& charge_fp,
        const amrex::Vector<std::unique_ptr<amrex::MultiFab>>& charge_cp,
        const amrex::Vector<std::unique_ptr<amrex::MultiFab>>& charge_buffer,
        const int lev,
        const int icomp,
        const int ncomp);
    void NodalSyncRho (
        const amrex::Vector<std::unique_ptr<amrex::MultiFab>>& charge_fp,
        const amrex::Vector<std::unique_ptr<amrex::MultiFab>>& charge_cp,
        const int lev,
        PatchType patch_type,
        const int icomp,
        const int ncomp);
 
    void ReadParameters ();
 
    void BackwardCompatibility ();
 
    void InitFromScratch ();
 
    void AllocLevelData (int lev, const amrex::BoxArray& new_grids,
                         const amrex::DistributionMapping& new_dmap);
 
    amrex::DistributionMapping
    GetRestartDMap (const std::string& chkfile, const amrex::BoxArray& ba, int lev) const;
 
    void InitFromCheckpoint ();
    void PostRestart ();
 
    void InitPML ();
    void ComputePMLFactors ();
 
    void InitFilter ();
 
    void InitDiagnostics ();
 
    void InitNCICorrector ();
 
    void CheckGuardCells();
 
    void CheckKnownIssues();
 
    void PerformanceHints ();
 
    void BuildBufferMasks ();
public: // for cuda
    void BuildBufferMasksInBox ( const amrex::Box tbx, amrex::IArrayBox &buffer_mask,
                                 const amrex::IArrayBox &guard_mask, const int ng );
private:
    const amrex::iMultiFab* getCurrentBufferMasks (int lev) const {
        return current_buffer_masks[lev].get();
    }
    const amrex::iMultiFab* getGatherBufferMasks (int lev) const {
        return gather_buffer_masks[lev].get();
    }
 
    void ReorderFornbergCoefficients (amrex::Vector<amrex::Real>& ordered_coeffs,
                                      amrex::Vector<amrex::Real>& unordered_coeffs,
                                      const int order);
    void AllocateCenteringCoefficients (amrex::Gpu::DeviceVector<amrex::Real>& device_centering_stencil_coeffs_x,
                                        amrex::Gpu::DeviceVector<amrex::Real>& device_centering_stencil_coeffs_y,
                                        amrex::Gpu::DeviceVector<amrex::Real>& device_centering_stencil_coeffs_z,
                                        const int centering_nox,
                                        const int centering_noy,
                                        const int centering_noz,
                                        const short a_grid_type);
 
    void AllocLevelMFs (int lev, const amrex::BoxArray& ba, const amrex::DistributionMapping& dm,
                        const amrex::IntVect& ngEB, amrex::IntVect& ngJ,
                        const amrex::IntVect& ngRho, const amrex::IntVect& ngF,
                        const amrex::IntVect& ngG, const bool aux_is_nodal);
 
#ifdef WARPX_USE_PSATD
#   ifdef WARPX_DIM_RZ
    void AllocLevelSpectralSolverRZ (amrex::Vector<std::unique_ptr<SpectralSolverRZ>>& spectral_solver,
                                     const int lev,
                                     const amrex::BoxArray& realspace_ba,
                                     const amrex::DistributionMapping& dm,
                                     const std::array<amrex::Real,3>& dx);
#   else
    void AllocLevelSpectralSolver (amrex::Vector<std::unique_ptr<SpectralSolver>>& spectral_solver,
                                   const int lev,
                                   const amrex::BoxArray& realspace_ba,
                                   const amrex::DistributionMapping& dm,
                                   const std::array<amrex::Real,3>& dx,
                                   const bool pml_flag=false);
#   endif
#endif
 
    amrex::Vector<int> istep;      // which step?
    amrex::Vector<int> nsubsteps;  // how many substeps on each level?
 
    amrex::Vector<amrex::Real> t_new;
    amrex::Vector<amrex::Real> t_old;
    amrex::Vector<amrex::Real> dt;
 
    // Particle container
    std::unique_ptr<MultiParticleContainer> mypc;
    std::unique_ptr<MultiDiagnostics> multi_diags;
 
    //
    // Fields: First array for level, second for direction
    //
 
    // Full solution
    amrex::Vector<std::array< std::unique_ptr<amrex::MultiFab>, 3 > > Efield_aux;
    amrex::Vector<std::array< std::unique_ptr<amrex::MultiFab>, 3 > > Bfield_aux;
 
    // Fine patch
    amrex::Vector<            std::unique_ptr<amrex::MultiFab>      > F_fp;
    amrex::Vector<            std::unique_ptr<amrex::MultiFab>      > G_fp;
    amrex::Vector<            std::unique_ptr<amrex::MultiFab>      > rho_fp;
    amrex::Vector<            std::unique_ptr<amrex::MultiFab>      > phi_fp;
    amrex::Vector<std::array< std::unique_ptr<amrex::MultiFab>, 3 > > current_fp;
    amrex::Vector<std::array< std::unique_ptr<amrex::MultiFab>, 3 > > current_fp_vay;
    amrex::Vector<std::array< std::unique_ptr<amrex::MultiFab>, 3 > > Efield_fp;
    amrex::Vector<std::array< std::unique_ptr<amrex::MultiFab>, 3 > > Bfield_fp;
    amrex::Vector<std::array< std::unique_ptr<amrex::MultiFab>, 3 > > Efield_avg_fp;
    amrex::Vector<std::array< std::unique_ptr<amrex::MultiFab>, 3 > > Bfield_avg_fp;
 
    // Memory buffers for computing magnetostatic fields
    // Vector Potential A and previous step.  Time buffer needed for computing dA/dt to first order
    amrex::Vector<std::array< std::unique_ptr<amrex::MultiFab>, 3 > > vector_potential_fp_nodal;
    amrex::Vector<std::array< std::unique_ptr<amrex::MultiFab>, 3 > > vector_potential_grad_buf_e_stag;
    amrex::Vector<std::array< std::unique_ptr<amrex::MultiFab>, 3 > > vector_potential_grad_buf_b_stag;
 
    // Same as Bfield_fp/Efield_fp for reading external field data
    amrex::Vector<std::array< std::unique_ptr<amrex::MultiFab>, 3 > > Efield_fp_external;
    amrex::Vector<std::array< std::unique_ptr<amrex::MultiFab>, 3 > > Bfield_fp_external;
 
    amrex::Vector<std::array< std::unique_ptr<amrex::MultiFab>, 3 > > m_edge_lengths;
    amrex::Vector<std::array< std::unique_ptr<amrex::MultiFab>, 3 > > m_face_areas;
 
    amrex::Vector<std::array< std::unique_ptr<amrex::iMultiFab>, 3 > > m_flag_info_face;
    amrex::Vector<std::array< std::unique_ptr<amrex::iMultiFab>, 3 > > m_flag_ext_face;
    amrex::Vector<std::array< std::unique_ptr<amrex::MultiFab>, 3 > > m_area_mod;
    amrex::Vector<std::array< std::unique_ptr<amrex::LayoutData<FaceInfoBox> >, 3 > > m_borrowing;
 
    amrex::Vector<std::array< std::unique_ptr<amrex::MultiFab>, 3 > > ECTRhofield;
    amrex::Vector<std::array< std::unique_ptr<amrex::MultiFab>, 3 > > Venl;
 
    //EB level set
    amrex::Vector<std::unique_ptr<amrex::MultiFab> > m_distance_to_eb;
 
    // store fine patch
    amrex::Vector<std::array< std::unique_ptr<amrex::MultiFab>, 3 > > current_store;
 
    // Nodal MultiFab for nodal current deposition if warpx.do_current_centering = 1
    amrex::Vector<std::array<std::unique_ptr<amrex::MultiFab>,3>> current_fp_nodal;
 
    // Coarse patch
    amrex::Vector<            std::unique_ptr<amrex::MultiFab>      > F_cp;
    amrex::Vector<            std::unique_ptr<amrex::MultiFab>      > G_cp;
    amrex::Vector<            std::unique_ptr<amrex::MultiFab>      > rho_cp;
    amrex::Vector<std::array< std::unique_ptr<amrex::MultiFab>, 3 > > current_cp;
    amrex::Vector<std::array< std::unique_ptr<amrex::MultiFab>, 3 > > Efield_cp;
    amrex::Vector<std::array< std::unique_ptr<amrex::MultiFab>, 3 > > Bfield_cp;
    amrex::Vector<std::array< std::unique_ptr<amrex::MultiFab>, 3 > > Efield_avg_cp;
    amrex::Vector<std::array< std::unique_ptr<amrex::MultiFab>, 3 > > Bfield_avg_cp;
 
    // Copy of the coarse aux
    amrex::Vector<std::array<std::unique_ptr<amrex::MultiFab>, 3 > > Efield_cax;
    amrex::Vector<std::array<std::unique_ptr<amrex::MultiFab>, 3 > > Bfield_cax;
    amrex::Vector<std::unique_ptr<amrex::iMultiFab> > current_buffer_masks;
    amrex::Vector<std::unique_ptr<amrex::iMultiFab> > gather_buffer_masks;
 
    // If charge/current deposition buffers are used
    amrex::Vector<std::array< std::unique_ptr<amrex::MultiFab>, 3 > > current_buf;
    amrex::Vector<std::unique_ptr<amrex::MultiFab> > charge_buf;
 
    // PML
    int do_pml = 0;
    int do_silver_mueller = 0;
    int pml_ncell = 10;
    int pml_delta = 10;
    int pml_has_particles = 0;
    int do_pml_j_damping = 0;
    int do_pml_in_domain = 0;
    static int do_similar_dm_pml;
    bool do_pml_dive_cleaning; // default set in WarpX.cpp
    bool do_pml_divb_cleaning; // default set in WarpX.cpp
    amrex::Vector<amrex::IntVect> do_pml_Lo;
    amrex::Vector<amrex::IntVect> do_pml_Hi;
    amrex::Vector<std::unique_ptr<PML> > pml;
#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD)
    amrex::Vector<std::unique_ptr<PML_RZ> > pml_rz;
#endif
    amrex::Real v_particle_pml;
 
    amrex::Real moving_window_x = std::numeric_limits<amrex::Real>::max();
 
    // Plasma injection parameters
    int warpx_do_continuous_injection = 0;
    int num_injected_species = -1;
    amrex::Vector<int> injected_plasma_species;
 
    std::optional<amrex::Real> m_const_dt;
 
    // Macroscopic properties
    std::unique_ptr<MacroscopicProperties> m_macroscopic_properties;
 
    // Hybrid PIC algorithm parameters
    std::unique_ptr<HybridPICModel> m_hybrid_pic_model;
 
    // Load balancing
    utils::parser::IntervalsParser load_balance_intervals;
    amrex::Vector<std::unique_ptr<amrex::LayoutData<amrex::Real> > > costs;
    int load_balance_with_sfc = 0;
    amrex::Real load_balance_knapsack_factor = amrex::Real(1.24);
    amrex::Real load_balance_efficiency_ratio_threshold = amrex::Real(1.1);
    amrex::Vector<amrex::Real> load_balance_efficiency;
    amrex::Real costs_heuristic_cells_wt = amrex::Real(0);
    amrex::Real costs_heuristic_particles_wt = amrex::Real(0);
 
    // Determines timesteps for override sync
    utils::parser::IntervalsParser override_sync_intervals;
 
    // Other runtime parameters
    int verbose = 1;
 
    bool use_hybrid_QED = 0;
 
    int max_step   = std::numeric_limits<int>::max();
    amrex::Real stop_time = std::numeric_limits<amrex::Real>::max();
 
    int regrid_int = -1;
 
    amrex::Real cfl = amrex::Real(0.999);
 
    std::string restart_chkfile;
 
    amrex::VisMF::Header::Version plotfile_headerversion  = amrex::VisMF::Header::Version_v1;
    amrex::VisMF::Header::Version slice_plotfile_headerversion  = amrex::VisMF::Header::Version_v1;
 
    bool use_single_read = true;
    bool use_single_write = true;
    int mffile_nstreams = 4;
    int field_io_nfiles = 1024;
    int particle_io_nfiles = 1024;
 
    amrex::RealVect fine_tag_lo;
    amrex::RealVect fine_tag_hi;
 
    bool is_synchronized = true;
 
    // Synchronization of nodal points
    static constexpr bool sync_nodal_points = true;
 
    guardCellManager guard_cells;
 
    //Slice Parameters
    int slice_max_grid_size;
    int slice_plot_int = -1;
    amrex::RealBox slice_realbox;
    amrex::IntVect slice_cr_ratio;
    amrex::Vector<            std::unique_ptr<amrex::MultiFab>      > F_slice;
    amrex::Vector<            std::unique_ptr<amrex::MultiFab>      > G_slice;
    amrex::Vector<            std::unique_ptr<amrex::MultiFab>      > rho_slice;
    amrex::Vector<std::array< std::unique_ptr<amrex::MultiFab>, 3 > > current_slice;
    amrex::Vector<std::array< std::unique_ptr<amrex::MultiFab>, 3 > > Efield_slice;
    amrex::Vector<std::array< std::unique_ptr<amrex::MultiFab>, 3 > > Bfield_slice;
 
    bool fft_periodic_single_box = false;
    int nox_fft = 16;
    int noy_fft = 16;
    int noz_fft = 16;
 
    amrex::IntVect numprocs{0};
 
    std::unique_ptr<ParticleBoundaryBuffer> m_particle_boundary_buffer;
 
    // Accelerator lattice elements
    amrex::Vector< std::unique_ptr<AcceleratorLattice> > m_accelerator_lattice;
 
    //
    // Embedded Boundary
    //
 
    // Factory for field data
    amrex::Vector<std::unique_ptr<amrex::FabFactory<amrex::FArrayBox> > > m_field_factory;
 
    amrex::FabFactory<amrex::FArrayBox> const& fieldFactory (int lev) const noexcept {
        return *m_field_factory[lev];
    }
#ifdef AMREX_USE_EB
public:
    amrex::EBFArrayBoxFactory const& fieldEBFactory (int lev) const noexcept {
        return static_cast<amrex::EBFArrayBoxFactory const&>(*m_field_factory[lev]);
    }
#endif
 
public:
    void InitEB ();
public:
#ifdef AMREX_USE_EB
    static void ComputeEdgeLengths (std::array< std::unique_ptr<amrex::MultiFab>, 3 >& edge_lengths,
                                    const amrex::EBFArrayBoxFactory& eb_fact);
    static void ComputeFaceAreas (std::array< std::unique_ptr<amrex::MultiFab>, 3 >& face_areas,
                                  const amrex::EBFArrayBoxFactory& eb_fact);
 
    static void ScaleEdges (std::array< std::unique_ptr<amrex::MultiFab>, 3 >& edge_lengths,
                            const std::array<amrex::Real,3>& cell_size);
    static void ScaleAreas (std::array< std::unique_ptr<amrex::MultiFab>, 3 >& face_areas,
                            const std::array<amrex::Real,3>& cell_size);
    void MarkCells();
#endif
    void ComputeDistanceToEB ();
    amrex::Array1D<int, 0, 2> CountExtFaces();
    void ComputeFaceExtensions();
    void InitBorrowing();
    void ShrinkBorrowing();
    void ComputeOneWayExtensions();
    void ComputeEightWaysExtensions();
    void ApplyBCKCorrection(const int idim);
 
    void PSATDSubtractCurrentPartialSumsAvg ();
 
private:
    void ScrapeParticles ();
 
    void PushPSATD ();
 
#ifdef WARPX_USE_PSATD
 
    void PSATDForwardTransformEB (
        const amrex::Vector<std::array<std::unique_ptr<amrex::MultiFab>,3>>& E_fp,
        const amrex::Vector<std::array<std::unique_ptr<amrex::MultiFab>,3>>& B_fp,
        const amrex::Vector<std::array<std::unique_ptr<amrex::MultiFab>,3>>& E_cp,
        const amrex::Vector<std::array<std::unique_ptr<amrex::MultiFab>,3>>& B_cp);
 
    void PSATDBackwardTransformEB (
        const amrex::Vector<std::array<std::unique_ptr<amrex::MultiFab>,3>>& E_fp,
        const amrex::Vector<std::array<std::unique_ptr<amrex::MultiFab>,3>>& B_fp,
        const amrex::Vector<std::array<std::unique_ptr<amrex::MultiFab>,3>>& E_cp,
        const amrex::Vector<std::array<std::unique_ptr<amrex::MultiFab>,3>>& B_cp);
 
    void PSATDBackwardTransformEBavg (
        const amrex::Vector<std::array<std::unique_ptr<amrex::MultiFab>,3>>& E_avg_fp,
        const amrex::Vector<std::array<std::unique_ptr<amrex::MultiFab>,3>>& B_avg_fp,
        const amrex::Vector<std::array<std::unique_ptr<amrex::MultiFab>,3>>& E_avg_cp,
        const amrex::Vector<std::array<std::unique_ptr<amrex::MultiFab>,3>>& B_avg_cp);
 
    void PSATDForwardTransformJ (
        const amrex::Vector<std::array<std::unique_ptr<amrex::MultiFab>,3>>& J_fp,
        const amrex::Vector<std::array<std::unique_ptr<amrex::MultiFab>,3>>& J_cp,
        const bool apply_kspace_filter=true);
 
    void PSATDBackwardTransformJ (
        const amrex::Vector<std::array<std::unique_ptr<amrex::MultiFab>,3>>& J_fp,
        const amrex::Vector<std::array<std::unique_ptr<amrex::MultiFab>,3>>& J_cp);
 
    void PSATDForwardTransformRho (
        const amrex::Vector<std::unique_ptr<amrex::MultiFab>>& charge_fp,
        const amrex::Vector<std::unique_ptr<amrex::MultiFab>>& charge_cp,
        const int icomp, const int dcomp, const bool apply_kspace_filter=true);
 
    void PSATDMoveRhoNewToRhoOld ();
 
    void PSATDMoveJNewToJOld ();
 
    void PSATDForwardTransformF ();
 
    void PSATDBackwardTransformF ();
 
    void PSATDForwardTransformG ();
 
    void PSATDBackwardTransformG ();
 
    void PSATDCurrentCorrection ();
 
    void PSATDVayDeposition ();
 
    void PSATDPushSpectralFields ();
 
    void PSATDScaleAverageFields (const amrex::Real scale_factor);
 
    void PSATDEraseAverageFields ();
 
#   ifdef WARPX_DIM_RZ
        amrex::Vector<std::unique_ptr<SpectralSolverRZ>> spectral_solver_fp;
        amrex::Vector<std::unique_ptr<SpectralSolverRZ>> spectral_solver_cp;
#   else
        amrex::Vector<std::unique_ptr<SpectralSolver>> spectral_solver_fp;
        amrex::Vector<std::unique_ptr<SpectralSolver>> spectral_solver_cp;
#   endif
 
public:
 
#   ifdef WARPX_DIM_RZ
        SpectralSolverRZ&
#   else
        SpectralSolver&
#   endif
            get_spectral_solver_fp (int lev) {return *spectral_solver_fp[lev];}
#endif
 
public:
    FiniteDifferenceSolver * get_pointer_fdtd_solver_fp (int lev) { return m_fdtd_solver_fp[lev].get(); }
 
private:
    amrex::Vector<std::unique_ptr<FiniteDifferenceSolver>> m_fdtd_solver_fp;
    amrex::Vector<std::unique_ptr<FiniteDifferenceSolver>> m_fdtd_solver_cp;
};
 
#endif