icache: Read iTLB using early next NIA from fetch1

Using i_in.next_nia means that we can read the iTLB RAM arrays
synchronously rather than asynchronously, which gives more opportunity
for using block RAMs in FPGA implementations.

The reading is gated by the stall signals because the next_nia can
advance when stalled, but we need the iTLB entry for the instruction
that i_in.nia points to.  If we are stalled because of an iTLB miss,
that means we don't see the new iTLB entry when it is written.
Instead we save the new entry directly when it arrives and use it
instead of the values read from the iTLB RAM.

Signed-off-by: Paul Mackerras <paulus@ozlabs.org>

icache.vhdl

@@ -176,8 +176,16 @@ architecture rtl of icache is
     signal itlb_valids : tlb_valids_t;
     signal itlb_tags : tlb_tags_t;
     signal itlb_ptes : tlb_ptes_t;
-    attribute ram_style of itlb_tags : signal is "distributed";
-    attribute ram_style of itlb_ptes : signal is "distributed";
+
+    -- Values read from above arrays on a clock edge
+    signal itlb_valid : std_ulogic;
+    signal itlb_ttag : tlb_tag_t;
+    signal itlb_pte : tlb_pte_t;
+
+    -- Values captured from a write to a TLB
+    signal itlb_bypass_valid : std_ulogic;
+    signal itlb_bypass_ra    : std_ulogic_vector(REAL_ADDR_BITS - TLB_LG_PGSZ - 1 downto 0);
+    signal itlb_bypass_priv  : std_ulogic;
 
     -- Privilege bit from PTE EAA field
     signal eaa_priv  : std_ulogic;
@@ -491,33 +499,61 @@ begin
         end process;
     end generate;
 
+    -- Read TLB using the NIA for the next cycle
+    itlb_read : process(clk)
+	variable tlb_req_index : std_ulogic_vector(TLB_BITS - 1 downto 0);
+    begin
+        if rising_edge(clk) then
+            if flush_in = '1' or i_in.req = '0' or (stall_in = '0' and stall_out = '0') then
+                tlb_req_index := hash_ea(i_in.next_nia);
+                if is_X(tlb_req_index) then
+                    itlb_pte <= (others => 'X');
+                    itlb_ttag <= (others => 'X');
+		    itlb_valid <= 'X';
+                else
+                    itlb_pte <= itlb_ptes(to_integer(unsigned(tlb_req_index)));
+                    itlb_ttag <= itlb_tags(to_integer(unsigned(tlb_req_index)));
+		    itlb_valid <= itlb_valids(to_integer(unsigned(tlb_req_index)));
+                end if;
+            end if;
+        end if;
+    end process;
+
+    -- Store TLB data being written for use in servicing the current request
+    itlb_bypass: process(clk)
+    begin
+        if rising_edge(clk) then
+            if rst = '1' then
+                itlb_bypass_valid <= '0';
+                itlb_bypass_ra <= (others => '0');
+                itlb_bypass_priv <= '0';
+            elsif flush_in = '1' or i_in.req = '0' or stall_out = '0' then
+                itlb_bypass_valid <= '0';
+            elsif m_in.tlbld = '1' then
+                assert i_in.nia(63 downto TLB_LG_PGSZ) = m_in.addr(63 downto TLB_LG_PGSZ);
+                itlb_bypass_valid <= '1';
+                itlb_bypass_ra <= m_in.pte(REAL_ADDR_BITS - 1 downto TLB_LG_PGSZ);
+                itlb_bypass_priv <= m_in.pte(3);
+            end if;
+        end if;
+    end process;
+
     -- TLB hit detection and real address generation
     itlb_lookup : process(all)
-        variable pte : tlb_pte_t;
-        variable ttag : tlb_tag_t;
-	variable tlb_req_index : std_ulogic_vector(TLB_BITS - 1 downto 0);
     begin
-        tlb_req_index := hash_ea(i_in.nia);
-	if is_X(tlb_req_index) then
-	    pte := (others => 'X');
-	    ttag := (others => 'X');
-	else
-	    pte := itlb_ptes(to_integer(unsigned(tlb_req_index)));
-	    ttag := itlb_tags(to_integer(unsigned(tlb_req_index)));
-	end if;
-        if i_in.virt_mode = '1' then
-            real_addr <= pte(REAL_ADDR_BITS - 1 downto TLB_LG_PGSZ) &
+        if itlb_bypass_valid = '1' then
+            real_addr <= itlb_bypass_ra & i_in.nia(TLB_LG_PGSZ - 1 downto 0);
+            ra_valid <= '1';
+            eaa_priv <= itlb_bypass_priv;
+        elsif i_in.virt_mode = '1' then
+            real_addr <= itlb_pte(REAL_ADDR_BITS - 1 downto TLB_LG_PGSZ) &
                          i_in.nia(TLB_LG_PGSZ - 1 downto 0);
-            if ttag = i_in.nia(63 downto TLB_LG_PGSZ + TLB_BITS) then
-		if is_X(tlb_req_index) then
-		    ra_valid <= 'X';
-		else
-		    ra_valid <= itlb_valids(to_integer(unsigned(tlb_req_index)));
-		end if;
+            if itlb_ttag = i_in.nia(63 downto TLB_LG_PGSZ + TLB_BITS) then
+                ra_valid <= itlb_valid;
             else
                 ra_valid <= '0';
             end if;
-            eaa_priv <= pte(3);
+            eaa_priv <= itlb_pte(3);
         else
             real_addr <= addr_to_real(i_in.nia);
             ra_valid <= '1';