#
# This file is part of LitePCIe.
#
# Copyright (c) 2015-2020 Florent Kermarrec <florent@enjoy-digital.fr>
# Copyright (c) 2020 Antmicro <www.antmicro.com>
# SPDX-License-Identifier: BSD-2-Clause

from migen import *

from litex.soc.interconnect import stream
from litex.soc.interconnect.csr import *

from litepcie.common import *
from litepcie.tlp.common import *

# Constants/Layouts --------------------------------------------------------------------------------

def descriptor_layout(with_user_id=False):
    layout = [("address", 32), ("length",  24), ("irq_disable", 1), ("last_disable", 1)]
    if with_user_id:
        layout += [("user_id", 8)]
    return EndpointDescription(layout)


# LitePCIeDMAScatterGather --------------------------------------------------------------------------

class LitePCIeDMAScatterGather(Module, AutoCSR):
    """LitePCIe DMA Scatter-Gather

    Software programmable table storing a list of DMA descriptors.

                               Mode
                                │
                              ┌─▼─┐  ┌───────────────────┐
       Descriptor to program  │   │  │                   │
             (Prog Mode) ─────►   │  │                   │
                              │   │  │    ScatterGather  │
                              │   ├──►     Table(FIFO)   ├──┬───► Descriptor (To DMA).
                          ┌───►   │  │                   │  │
                          │   │   │  │                   │  │
                          │   └───┘  └───────────────────┘  │
                          │                                 │
                          └─────────────────────────────────┘
                                    Refill (Loop Mode)


    A DMA descriptor is composed of:
    - a 32-bit address: The base address of the Host where the data stream should be written/read.
    - a 24-bit length : The length of the data stream (bytes).
    - a 8-bit control : Dynamic controls (ex: Disable IRQ generation, disable Last handling).

    The table is implemented as a FIFO initially filled by software. Once enabled, the DMA gets the
    descriptors from this table and executes them.

    This module has two modes:
    - Prog mode: Used to program the table by software and for cases where automatic refill of the
    table is not needed: A descriptor is only executed once and when all the descriptors have been
    executed (ie the table is empty), the DMA just stops until the next software refill.
    - Loop mode: Used once the table has been filled by software in PROG mode and allow continuous
    Scatter-Gather DMA: Each descriptor sent to the DMA is refilled to the table.

    In Loop mode, a loop status is maintained by the hardware for the software synchronization of the
    DMA buffers. (Even if a MSI IRQ is generated after a descriptor has been executed, since IRQ can
    potentially be lost, it's safer for the software to just use the hardware loop status than to
    maintain a software loop status based MSI IRQ reception).
    """
    def __init__(self, depth):
        # Stream Endpoint.
        self.source = source = stream.Endpoint(descriptor_layout())

        # Control/Status.
        self.value = CSRStorage(64, reset_less=True, fields=[
            CSRField("address",      size=32, description="32-bit Address of the descriptor (bytes-aligned)."),
            CSRField("length",       size=24, description="24-bit Length  of the descriptor (in bytes)."),
            CSRField("irq_disable",  size=1,  description="IRQ Disable Control of the descriptor."),
            CSRField("last_disable", size=1,  description="Last Disable Control of the descriptor.")
            ], description="64-bit DMA descriptor to be written to the table.")
        self.we = CSRStorage(description="A write to this register adds the descriptor to table.")
        self.loop_prog_n = CSRStorage(description="""Mode Selection.\n
            ``0``: **Prog** mode / ``1``: **Loop** mode.\n
            **Prog** mode should be used to program the table by software and for cases where automatic
            refill of the table is not needed: A descriptor is only executed once and when all the
            descriptors have been executed (ie the table is empty), the DMA just stops until the next
            software refill.\n
            **Loop** mode should be used once the table has been filled by software in **Prog** mode
            and allow continuous Scatter-Gather DMA: Each descriptor sent to the DMA is refilled to the table.
            """)
        self.loop_status = CSRStatus(fields=[
            CSRField("index", size=16, description= "Index of the last descriptor executed in the DMA descriptor table."),
            CSRField("count", size=16, description= "Loops of the DMA descriptor table since started."),
            ], description="Loop monitoring for software synchronization.")
        self.level = CSRStatus(bits_for(depth), description="Number descriptors in the table.")
        self.reset = CSRStorage(description="A write to this register resets the table.")

        # # #

        # Table (FIFO) -----------------------------------------------------------------------------
        table = stream.SyncFIFO(descriptor_layout(), depth)
        table = ResetInserter()(table)
        self.submodules += table
        self.comb += table.reset.eq(self.reset.storage & self.reset.re)
        self.comb += self.level.status.eq(table.level)

        # Table Write logic ------------------------------------------------------------------------
        prog_mode = (self.loop_prog_n.storage == 0)
        self.sync += [
            # In Prog mode, the Table is filled through the CSRs.
            If(prog_mode,
                table.sink.address.eq(self.value.fields.address),
                table.sink.length.eq(self.value.fields.length),
                table.sink.irq_disable.eq(self.value.fields.irq_disable),
                table.sink.last_disable.eq(self.value.fields.last_disable),
                table.sink.first.eq(table.level == 0),
                table.sink.valid.eq(self.we.storage & self.we.re),
            # In Loop mode, the Table is automatically refilled.
            ).Else(
                table.source.connect(table.sink, omit={"valid", "ready"}),
                table.sink.valid.eq(table.source.valid & table.source.ready),
            )
        ]

        # Table Read logic -------------------------------------------------------------------------
        self.comb += table.source.connect(source)

        # Loop Status (For Software Sychronization in Loop mode) -----------------------------------
        loop_index = self.loop_status.fields.index
        loop_count = self.loop_status.fields.count
        self.sync += [
            # Reset Loop Index/Count on Table reset.
            If(table.reset,
                # Init count to 2**n-1 to have (index, count) == (0, 0) for the first Descriptor.
                loop_index.eq(0),
                loop_count.eq(2**len(loop_count) - 1),
            # When a Descriptor is consumned...
            ).Elif(table.source.valid & table.source.ready,
                # Update Loop Status with current Loop Index/Count.
                # Update Loop Index/Count.
                If(table.source.first,
                    loop_index.eq(0),
                    loop_count.eq(loop_count + 1),
                ).Else(
                    loop_index.eq(loop_index + 1)
                )
            )
        ]

# LitePCIeDMADescriptorSplitter --------------------------------------------------------------------

class LitePCIeDMADescriptorSplitter(Module, AutoCSR):
    """LitePCIe DMA Descriptor Splitter

    Splits descriptors from LitePCIeDMAScatterGather in shorter descriptors of:
    - Maximum Payload Size for Writes.
    - Maximum Request Size for Reads.

    Descriptors from LitePCIeDMAScatterGather have a maximum length of 16Mb (24-bits). It is not
    possible to do such long Writes/Reads on the PCIe bus. At the PCIe enumeration, Maximum Payload
    and Request Sizes are negotiated between the Host and the Device. Writes are limited to Maximum
    Payload Size, Reads are limited to Maximum Request Size. Each descriptor is then split in
    several shorter descriptors.
    """
    def __init__(self, max_size):
        # Stream Endpoints.
        self.sink   =   sink = stream.Endpoint(descriptor_layout())
        self.source = source = stream.Endpoint(descriptor_layout(with_user_id=True))

        # Control.
        self.end    = Signal()

        # # #

        desc_length  = Signal(32)
        desc_offset  = Signal(32)
        desc_id      = Signal(32)

        # FSM --------------------------------------------------------------------------------------
        self.submodules.fsm = fsm = FSM(reset_state="IDLE")
        fsm.act("IDLE",
            # Set Descriptor Offset/Length.
            NextValue(desc_offset, 0),
            NextValue(desc_length, sink.length),
            # Wait for a Descriptor and go to Run.
            If(sink.valid,
                NextState("SPLIT")
            )
        )
        # Split Data-Path.
        self.comb += [
            source.address.eq(sink.address + desc_offset),
            source.irq_disable.eq(sink.irq_disable),
            source.last_disable.eq(sink.last_disable),
            source.user_id.eq(desc_id),
        ]
        fsm.act("SPLIT",
            # Split Control-Path.
            source.valid.eq(1),
            source.first.eq(desc_offset == 0),
            # Full Descriptor when Length > max_size.
            If(desc_length > max_size,
                source.last.eq(self.end),
                source.length.eq(max_size),
            # Partial Descriptor when Length <= max_size
            ).Else(
                source.last.eq(1),
                source.length.eq(desc_length),
            ),
            # When Descriptor is accepted...
            If(source.ready,
                # Increment Offset.
                NextValue(desc_offset, desc_offset + max_size),
                # Decrement Length.
                NextValue(desc_length, desc_length - max_size),
                # When Last....
                If(source.last,
                    # Accept Descriptor.
                    sink.ready.eq(1),
                    # Increment ID.
                    NextValue(desc_id, desc_id + 1),
                    # Go to Idle.
                    NextState("IDLE")
                )
            )
        )

# LitePCIeDMAReader --------------------------------------------------------------------------------

class LitePCIeDMAReader(Module, AutoCSR):
    """LitePCIe DMA Reader

    Generates a data stream from Host's memory.

    This module allows Scatter-Gather DMAs from Host's memory to data stream in the FPGA. The DMA
    descriptors, stored in a software programmable table, are split and executed as Read Requests
    on the PCIe bus.

    A Read Request is only sent to the Host when enough space is available in the Data FIFO to store
    the requested data.

    A MSI IRQ can be generated when a descriptor has been executed.
    """
    def __init__(self, endpoint, port, with_table=True, table_depth=256):
        self.port = port
        # Stream Endpoint.
        self.source = stream.Endpoint(dma_layout(endpoint.phy.data_width))

        # Control.
        self.enable = CSRStorage(description="DMA Reader Control. Write ``1`` to enable DMA Reader.", reset=0 if with_table else 1)

        # IRQ.
        self.irq = Signal()

        # # #

        # CSR/Parameters ---------------------------------------------------------------------------
        enable = self.enable.storage

        length_shift          = log2_int(endpoint.phy.data_width//8)
        max_words_per_request = max_request_size//(endpoint.phy.data_width//8)
        max_pending_words     = endpoint.max_pending_requests*max_words_per_request

        # Table ------------------------------------------------------------------------------------
        if with_table:
            self.submodules.table = LitePCIeDMAScatterGather(table_depth)
        else:
            self.desc_sink = stream.Endpoint(descriptor_layout()) # Expose a Descriptor sink.

        # Splitter ---------------------------------------------------------------------------------
        # DMA descriptors need to be splitted in descriptors of max_request_size (negotiated at link-up)
        splitter = LitePCIeDMADescriptorSplitter(max_size=endpoint.phy.max_request_size)
        splitter = ResetInserter()(splitter)
        splitter = BufferizeEndpoints({"source": DIR_SOURCE})(splitter) # For timings.
        self.submodules.splitter = splitter
        if with_table:
            self.comb += self.table.source.connect(splitter.sink)
        else:
            self.comb += self.desc_sink.connect(splitter.sink)

        # User ID ----------------------------------------------------------------------------------
        last_user_id = Signal(8, reset=255)
        self.sync += If(port.sink.valid & port.sink.first & port.sink.ready,
            last_user_id.eq(port.sink.user_id)
        )

        # Data FIFO --------------------------------------------------------------------------------
        data_fifo_depth = 4*max_pending_words
        data_fifo = SyncFIFO(dma_layout(endpoint.phy.data_width), data_fifo_depth, buffered=True)
        self.submodules += ResetInserter()(data_fifo)
        self.comb += [
            # Connect Data FIFO to Source.
            data_fifo.source.connect(self.source),
            # When Enabled, connect Sink to Data FIFO.
            If(enable,
                port.sink.connect(data_fifo.sink, keep={"valid", "ready"}),
                data_fifo.sink.data.eq(port.sink.dat),
                data_fifo.sink.first.eq(port.sink.first & (port.sink.user_id != last_user_id)),
            # Else accept incoming Port Data.
            ).Else(
                port.sink.ready.eq(1)
            )
        ]

        # Pending words ----------------------------------------------------------------------------
        pending_words         = Signal(max=data_fifo_depth + 1)
        pending_words_queue   = Signal.like(pending_words)
        pending_words_dequeue = Signal.like(pending_words)
        self.comb += [
            # Queue Pending words as Read Requests are emitted.
            If(splitter.source.valid & splitter.source.ready,
                pending_words_queue.eq(splitter.source.length[length_shift:])
            ),
            # Dequeue Pending words as Read Responses are received.
            If(data_fifo.source.valid & data_fifo.source.ready,
                pending_words_dequeue.eq(1)
            ),
        ]
        # Update Pending words.
        self.sync += pending_words.eq(pending_words + pending_words_queue - pending_words_dequeue)
        self.sync += If(~enable, pending_words.eq(0))

        # FSM --------------------------------------------------------------------------------------
        self.submodules.fsm = fsm = FSM(reset_state="IDLE")
        fsm.act("IDLE",
            # Reset Splitter/FIFO when Disabled.
            splitter.reset.eq( ~enable),
            data_fifo.reset.eq(~enable),
            # Wait for a Descriptor and to have enough Space to generate the Request.
            If(splitter.source.valid & (pending_words < (data_fifo_depth - max_words_per_request)),
                NextState("REQUEST"),
            )
        )
        # Request Data-Path.
        self.comb += [
            port.source.channel.eq(port.channel),
            port.source.user_id.eq(splitter.source.user_id),
            port.source.first.eq(1),
            port.source.last.eq(1),
            port.source.we.eq(0),
            port.source.adr.eq(splitter.source.address),
            port.source.len.eq(splitter.source.length[2:]),
            port.source.req_id.eq(endpoint.phy.id),
            port.source.dat.eq(0),
        ]
        fsm.act("REQUEST",
            # Request Control-Path.
            port.source.valid.eq(1),
            # When Request is accepted...
            If(port.source.ready,
                # Accept Descriptor.
                splitter.source.ready.eq(1),
                # Return to Idle.
                NextState("IDLE"),
            )
        )

        # IRQ --------------------------------------------------------------------------------------
        self.comb += If(splitter.source.valid & splitter.source.ready & splitter.source.last,
            self.irq.eq(~splitter.source.irq_disable)
        )

# LitePCIeDMAWriter --------------------------------------------------------------------------------

class LitePCIeDMAWriter(Module, AutoCSR):
    """LitePCIe DMA Writer

    Stores a data stream to Host's memory.

    This module allows Scatter-Gather DMAs from a data stream in the FPGA to Host's memory. The DMA
    descriptors, stored in a software programmable table, are split and executed as Write Requests
    on the PCIe bus.

    A Write Request is only sent to the Host when enough data are available for the current split
    descriptor.

    A MSI IRQ can be generated when a descriptor has been executed.
    """
    def __init__(self, endpoint, port, with_table=True, table_depth=256):
        self.port = port
        # Stream Endpoint.
        self.sink   = sink = stream.Endpoint(dma_layout(endpoint.phy.data_width))

        # Control.
        self.enable = CSRStorage(description="DMA Writer Control. Write ``1`` to enable DMA Writer.", reset=0 if with_table else 1)

        # IRQ.
        self.irq = Signal()

        # # #

        # CSR/Parameters ---------------------------------------------------------------------------
        enable = self.enable.storage

        length_shift          = log2_int(endpoint.phy.data_width//8)
        max_words_per_request = max_payload_size//(endpoint.phy.data_width//8)

        # Table ------------------------------------------------------------------------------------
        if with_table:
            self.submodules.table = LitePCIeDMAScatterGather(table_depth)
        else:
            self.desc_sink = stream.Endpoint(descriptor_layout()) # Expose a Descriptor sink.

        # Splitter ---------------------------------------------------------------------------------
        # DMA descriptors need to be splitted in descriptors of max_request_size (negotiated at link-up)
        splitter = LitePCIeDMADescriptorSplitter(max_size=endpoint.phy.max_payload_size)
        splitter = ResetInserter()(splitter)
        splitter = BufferizeEndpoints({"source": DIR_SOURCE})(splitter) # For timings.
        self.submodules.splitter = splitter
        if with_table:
            self.comb += self.table.source.connect(splitter.sink)
        else:
            self.comb += self.desc_sink.connect(splitter.sink)

        # Data FIFO --------------------------------------------------------------------------------
        data_fifo_depth = 4*max_words_per_request
        data_fifo = stream.SyncFIFO([("data", endpoint.phy.data_width)], data_fifo_depth, buffered=True)
        self.submodules += ResetInserter()(data_fifo)
        self.comb += [
            # When Enabled, connect Sink to Data FIFO.
            If(enable,
                sink.connect(data_fifo.sink)
            # Else accept incoming Sink Data.
            ).Else(
                sink.ready.eq(1)
            )
        ]

        # FSM --------------------------------------------------------------------------------------
        req_count = Signal.like(splitter.source.length)
        self.submodules.fsm = fsm = FSM(reset_state="IDLE")
        fsm.act("IDLE",
            # Reset Splitter/FIFO when Disabled.
            splitter.reset.eq( ~enable),
            data_fifo.reset.eq(~enable),
            # Reset Request Count.
            NextValue(req_count, 0),
            # Wait for a Descriptor and to have enough Data to generate the Request.
            If(splitter.source.valid & (data_fifo.level >= splitter.source.length[length_shift:]),
                NextState("REQUEST"),
            )
        )
        # Request Data-Path.
        self.comb += [
            port.source.channel.eq(port.channel),
            port.source.user_id.eq(splitter.source.user_id),
            port.source.first.eq(req_count == 0),
            port.source.last.eq((req_count == splitter.source.length[length_shift:] - 1)),
            port.source.we.eq(1),
            port.source.adr.eq(splitter.source.address),
            port.source.req_id.eq(endpoint.phy.id),
            port.source.tag.eq(0),
            port.source.len.eq(splitter.source.length[2:]),
            port.source.dat.eq(data_fifo.source.data)
        ]
        fsm.act("REQUEST",
            # Request Control-Path.
            port.source.valid.eq(1),
            # When Request is accepted...
            If(port.source.ready,
                # Increment Request Count.
                NextValue(req_count, req_count + 1),
                # Accept Data.
                data_fifo.source.ready.eq(1),
                # When last...
                If(port.source.last,
                    # Accept Descriptor.
                    splitter.source.ready.eq(1),
                    # Early Splitter End (can be disabled).
                    splitter.end.eq(data_fifo.source.last & ~splitter.source.last_disable),
                    # Return to Idle.
                    NextState("IDLE"),
                )
            )
        )

        # IRQ --------------------------------------------------------------------------------------
        self.comb += If(splitter.source.valid & splitter.source.ready & splitter.source.last,
            self.irq.eq(~splitter.source.irq_disable)
        )

# LitePCIeDMALoopback ------------------------------------------------------------------------------

class LitePCIeDMALoopback(Module, AutoCSR):
    """LitePCIe DMA Loopback

    Optional DMA Reader to DMA Writer loopback.

    For software development or system bring-up/check, being able to do a DMA loopback in the FPGA
    is very useful. This module allows doing a DMA Reader to DMA Writer loopback that can be enabled
    by a CSR. When enabled, user data stream from the DMA Reader is no longer generated, the same
    goes for user data stream to the DMA Writer that is no longer consumed.
    """
    def __init__(self, data_width):
        self.enable      = CSRStorage(description="""DMA Loopback Enable Control.\n
         Write ``1`` to enable DMA internal loopback (DMA Reader to DMA Writer).""")

        self.sink        = stream.Endpoint(dma_layout(data_width))
        self.source      = stream.Endpoint(dma_layout(data_width))

        self.next_source = stream.Endpoint(dma_layout(data_width))
        self.next_sink   = stream.Endpoint(dma_layout(data_width))

        # # #

        self.comb += [
            If(self.enable.storage,
                self.sink.connect(self.source)
            ).Else(
                self.sink.connect(self.next_source),
                self.next_sink.connect(self.source)
            )
        ]

# LitePCIeDMASynchronizer --------------------------------------------------------------------------

class LitePCIeDMASynchronizer(Module, AutoCSR):
    """LitePCIe DMA Synchronizer

    Optional DMA synchronization.

    For some applications (Software Defined Radio, Video, ...), DMA start needs to be precisely
    synchronized to an internal signal of the FPGA (PPS for example for an SDR applications). This
    module allows releasing precisely the DMA Writer/Reader data streams.
    """
    def __init__(self, data_width):
        self.bypass      = CSRStorage()
        self.enable      = CSRStorage()
        self.ready       = Signal(reset=1)
        self.pps         = Signal()

        self.sink        = stream.Endpoint(dma_layout(data_width))
        self.source      = stream.Endpoint(dma_layout(data_width))

        self.next_source = stream.Endpoint(dma_layout(data_width))
        self.next_sink   = stream.Endpoint(dma_layout(data_width))

        # # #

        self.synced = synced = Signal()

        self.sync += [
            If(~self.enable.storage,
                synced.eq(0)
            ).Else(
                If(self.ready & self.sink.valid & (self.pps | self.bypass.storage),
                    synced.eq(1)
                )
            )
        ]
        self.comb += [
            If(synced,
                self.sink.connect(self.next_source),
                self.next_sink.connect(self.source),
            ).Else(
                # Block sink
                self.next_source.valid.eq(0),
                self.sink.ready.eq(0),

                # Ack next_sink
                self.source.valid.eq(0),
                self.next_sink.ready.eq(1),
            )
        ]

# LitePCIeDMABuffering -----------------------------------------------------------------------------

class LitePCIeDMABuffering(Module, AutoCSR):
    """LitePCIe DMA Buffering

    Optional DMA buffering with dynamically configurable depth.

    For some applications (Software Defined Radio, Video, ...), the user module consuming the data
    from the DMA Reader works at fixed rate and does not handle backpressure. (The same also applies
    to the user module generating the data to the DMA Writer). Since the PCIe bus is shared, gaps
    appears in the streams and our Writes/Reads can't be absorbed/produced at a fixed rate. A minimum
    of buffering is needed to make sure the gaps are smoothed and not propagated to user modules.
    """
    def __init__(self, data_width, writer_depth, reader_depth, dynamic_depth=True):
        self.sink        = stream.Endpoint(dma_layout(data_width))
        self.source      = stream.Endpoint(dma_layout(data_width))

        self.next_source = stream.Endpoint(dma_layout(data_width))
        self.next_sink   = stream.Endpoint(dma_layout(data_width))

        # Reader FIFO Control/Status.
        assert bits_for(reader_depth) < 24
        self.reader_fifo_control = CSRStorage(fields=[
            CSRField("depth", offset=0, size=bits_for(reader_depth), reset=reader_depth,
                description="DMA Reader FIFO depth (in {}-bit words).".format(data_width)),
            CSRField("scratch",    offset=24, size=4, description="Software Scratchpad."),
            CSRField("level_mode", offset=31, values=[
                ("``0b0``", "Report Instantaneous level."),
                ("``0b1``", "Report `Minimal` level since last clear.")
            ])
        ])
        self.reader_fifo_status = CSRStatus(fields=[
            CSRField("level", offset=0, size=bits_for(reader_depth),
                description="DMA Reader FIFO level (in {}-bit words).".format(data_width))
            ])

        # Writer FIFO Control/Status.
        assert bits_for(writer_depth) < 24
        self.writer_fifo_control = CSRStorage(fields=[
            CSRField("depth", offset=0, size=bits_for(writer_depth), reset=writer_depth,
                description="DMA Writer FIFO depth (in {}-bit words).".format(data_width)),
            CSRField("scratch",    offset=24, size=4, description="Software Scratchpad."),
            CSRField("level_mode", offset=31, values=[
                ("``0b0``", "Report Instantaneous level."),
                ("``0b1``", "Report `Maximal` level since last clear.")
            ])
        ])
        self.writer_fifo_status = CSRStatus(fields=[
            CSRField("level", offset=0, size=bits_for(reader_depth),
                description="DMA Writer FIFO level (in {}-bit words).".format(data_width))
            ])

        # # #

        depth_shift = log2_int(data_width//8)

        # Reader FIFO.
        reader_fifo = SyncFIFO(dma_layout(data_width), reader_depth//(data_width//8), buffered=True)
        self.submodules += reader_fifo
        self.comb += [
            # Connect Reader Sink to Reader FIFO when Level < Configured Depth.
            self.sink.connect(reader_fifo.sink, omit={"valid", "ready"}),
            If((reader_fifo.level < self.reader_fifo_control.fields.depth[depth_shift:]) | (not dynamic_depth),
                self.sink.connect(reader_fifo.sink, keep={"valid", "ready"})
            ),
            # Connect Reader FIFO to Reader Source.
            reader_fifo.source.connect(self.next_source),
        ]

        # Store Min.
        reader_fifo_level_min = Signal.like(reader_fifo.level)
        self.sync += If(reader_fifo.level < reader_fifo_level_min, reader_fifo_level_min.eq(reader_fifo.level))
        # Clear on Status write or when in Instantaneous mode.
        reader_fifo_level_clr = (self.reader_fifo_status.re | (self.reader_fifo_control.fields.level_mode == 0))
        self.sync += If(reader_fifo_level_clr, reader_fifo_level_min.eq(2**len(reader_fifo_level_min)-1))
        # Return Reader FIFO level.
        self.comb += [
            # Instantaneous.
            If(self.reader_fifo_control.fields.level_mode == 0,
                self.reader_fifo_status.fields.level[depth_shift:].eq(reader_fifo.level)
            # Min.
            ).Else(
                self.reader_fifo_status.fields.level[depth_shift:].eq(reader_fifo_level_min)
            )
        ]

        # Writer FIFO.
        writer_fifo = SyncFIFO(dma_layout(data_width), writer_depth//(data_width//8), buffered=True)
        self.submodules += writer_fifo
        self.comb += [
            # Connect Writer Sink to Writer FIFO when Level < Configured Depth.
            self.next_sink.connect(writer_fifo.sink, omit={"valid", "ready"}),
            If((writer_fifo.level < self.writer_fifo_control.fields.depth[depth_shift:]) | (not dynamic_depth),
                self.next_sink.connect(writer_fifo.sink, keep={"valid", "ready"})
            ),
            # Connect Writer FIFO to Writer Source.
            writer_fifo.source.connect(self.source),
        ]
        # Store Max.
        writer_fifo_level_max = Signal.like(writer_fifo.level)
        self.sync += If(writer_fifo.level > writer_fifo_level_max, writer_fifo_level_max.eq(writer_fifo.level))
        # Clear on Status write or when in Instantaneous mode.
        writer_fifo_level_clr = (self.writer_fifo_status.re | (self.writer_fifo_control.fields.level_mode == 0))
        self.sync += If(writer_fifo_level_clr, writer_fifo_level_max.eq(0))
        # Return Writer FIFO level.
        self.comb += [
            # Instantaneous.
            If(self.writer_fifo_control.fields.level_mode == 0,
                self.writer_fifo_status.fields.level[depth_shift:].eq(writer_fifo.level)
            # Min.
            ).Else(
                self.writer_fifo_status.fields.level[depth_shift:].eq(writer_fifo_level_max)
            )
        ]

# LitePCIeDMA --------------------------------------------------------------------------------------

class LitePCIeDMA(Module, AutoCSR):
    """LitePCIe DMA

    Scatter-Gather bi-directional DMA:
    - Generates a data stream from Host's memory.
    - Stores a data stream to Host's memory.

    Optional buffering, loopback, synchronization and monitoring.
    """
    def __init__(self, phy, endpoint, table_depth=256,
        with_loopback      = False,
        with_synchronizer  = False,
        with_buffering     = False, buffering_depth=256*8, writer_buffering_depth=None, reader_buffering_depth=None,
        with_monitor       = False):

        # Writer/Reader ----------------------------------------------------------------------------
        writer = LitePCIeDMAWriter(
            endpoint    = endpoint,
            port        = endpoint.crossbar.get_master_port(write_only=True),
            table_depth = table_depth,
        )
        reader = LitePCIeDMAReader(
            endpoint    = endpoint,
            port        = endpoint.crossbar.get_master_port(read_only=True),
            table_depth = table_depth,
        )
        self.submodules.writer = writer
        self.submodules.reader = reader
        self.sink, self.source = writer.sink, reader.source

        # Loopback ---------------------------------------------------------------------------------
        if with_loopback:
            self.submodules.loopback = LitePCIeDMALoopback(phy.data_width)
            self.add_plugin_module(self.loopback)

        # Synchronizer -----------------------------------------------------------------------------
        if with_synchronizer:
            self.submodules.synchronizer = LitePCIeDMASynchronizer(phy.data_width)
            self.add_plugin_module(self.synchronizer)

        # Buffering --------------------------------------------------------------------------------
        if with_buffering:
            writer_depth = writer_buffering_depth if writer_buffering_depth is not None else buffering_depth
            reader_depth = reader_buffering_depth if reader_buffering_depth is not None else buffering_depth
            self.submodules.buffering = LitePCIeDMABuffering(
                data_width   = phy.data_width,
                writer_depth = writer_depth,
                reader_depth = reader_depth)
            self.add_plugin_module(self.buffering)

        # Monitor ----------------------------------------------------------------------------------
        if with_monitor:
            self.submodules.writer_monitor = stream.Monitor(self.sink,   count_width=16, with_overflows  = True)
            self.submodules.reader_monitor = stream.Monitor(self.source, count_width=16, with_underflows = True)

    def add_plugin_module(self, m):
        self.comb += [
            self.source.connect(m.sink),
            m.source.connect(self.sink)
        ]
        self.sink, self.source = m.next_sink, m.next_source
