@@ -726,11 +726,17 @@ extern unsigned long pte_file(pte_t);
extern pte_t pgoff_to_pte(unsigned long);
#define PTE_FILE_MAX_BITS (64UL - PAGE_SHIFT - 1UL)
-extern unsigned long *sparc64_valid_addr_bitmap;
+extern unsigned long sparc64_valid_addr_bitmap[];
/* Needs to be defined here and not in linux/mm.h, as it is arch dependent */
-#define kern_addr_valid(addr) \
- (test_bit(__pa((unsigned long)(addr))>>22, sparc64_valid_addr_bitmap))
+static inline bool kern_addr_valid(unsigned long addr)
+{
+ unsigned long paddr = __pa(addr);
+
+ if ((paddr >> 41UL) != 0UL)
+ return false;
+ return test_bit(paddr >> 22, sparc64_valid_addr_bitmap);
+}
extern int page_in_phys_avail(unsigned long paddr);
@@ -151,12 +151,46 @@ kvmap_dtlb_4v:
* Must preserve %g1 and %g6 (TAG).
*/
kvmap_dtlb_tsb4m_miss:
- sethi %hi(kpte_linear_bitmap), %g2
- or %g2, %lo(kpte_linear_bitmap), %g2
+ /* Clear the PAGE_OFFSET top virtual bits, shift
+ * down to get PFN, and make sure PFN is in range.
+ */
+ sllx %g4, 21, %g5
- /* Clear the PAGE_OFFSET top virtual bits, then shift
- * down to get a 256MB physical address index.
+ /* Check to see if we know about valid memory at the 4MB
+ * chunk this physical address will reside within.
*/
+ srlx %g5, 21 + 41, %g2
+ brnz,pn %g2, kvmap_dtlb_longpath
+ nop
+
+ /* This unconditional branch and delay-slot nop gets patched
+ * by the sethi sequence once the bitmap is properly setup.
+ */
+ .globl valid_addr_bitmap_insn
+valid_addr_bitmap_insn:
+ ba,pt %xcc, 2f
+ nop
+ .subsection 2
+ .globl valid_addr_bitmap_patch
+valid_addr_bitmap_patch:
+ sethi %hi(sparc64_valid_addr_bitmap), %g7
+ or %g7, %lo(sparc64_valid_addr_bitmap), %g7
+ .previous
+
+ srlx %g5, 21 + 22, %g2
+ srlx %g2, 6, %g5
+ and %g2, 63, %g2
+ sllx %g5, 3, %g5
+ ldx [%g7 + %g5], %g5
+ mov 1, %g7
+ sllx %g7, %g2, %g7
+ andcc %g5, %g7, %g0
+ be,pn %xcc, kvmap_dtlb_longpath
+
+2: sethi %hi(kpte_linear_bitmap), %g2
+ or %g2, %lo(kpte_linear_bitmap), %g2
+
+ /* Get the 256MB physical address index. */
sllx %g4, 21, %g5
mov 1, %g7
srlx %g5, 21 + 28, %g5
@@ -145,7 +145,8 @@ static void __init read_obp_memory(const char *property,
cmp_p64, NULL);
}
-unsigned long *sparc64_valid_addr_bitmap __read_mostly;
+unsigned long sparc64_valid_addr_bitmap[VALID_ADDR_BITMAP_BYTES /
+ sizeof(unsigned long)];
EXPORT_SYMBOL(sparc64_valid_addr_bitmap);
/* Kernel physical address base and size in bytes. */
@@ -1874,7 +1875,7 @@ static int pavail_rescan_ents __initdata;
* memory list again, and make sure it provides at least as much
* memory as 'pavail' does.
*/
-static void __init setup_valid_addr_bitmap_from_pavail(void)
+static void __init setup_valid_addr_bitmap_from_pavail(unsigned long *bitmap)
{
int i;
@@ -1897,8 +1898,7 @@ static void __init setup_valid_addr_bitmap_from_pavail(void)
if (new_start <= old_start &&
new_end >= (old_start + PAGE_SIZE)) {
- set_bit(old_start >> 22,
- sparc64_valid_addr_bitmap);
+ set_bit(old_start >> 22, bitmap);
goto do_next_page;
}
}
@@ -1919,20 +1919,21 @@ static void __init setup_valid_addr_bitmap_from_pavail(void)
}
}
+static void __init patch_tlb_miss_handler_bitmap(void)
+{
+ extern unsigned int valid_addr_bitmap_insn[];
+ extern unsigned int valid_addr_bitmap_patch[];
+
+ valid_addr_bitmap_insn[1] = valid_addr_bitmap_patch[1];
+ mb();
+ valid_addr_bitmap_insn[0] = valid_addr_bitmap_patch[0];
+ flushi(&valid_addr_bitmap_insn[0]);
+}
+
void __init mem_init(void)
{
unsigned long codepages, datapages, initpages;
unsigned long addr, last;
- int i;
-
- i = last_valid_pfn >> ((22 - PAGE_SHIFT) + 6);
- i += 1;
- sparc64_valid_addr_bitmap = (unsigned long *) alloc_bootmem(i << 3);
- if (sparc64_valid_addr_bitmap == NULL) {
- prom_printf("mem_init: Cannot alloc valid_addr_bitmap.\n");
- prom_halt();
- }
- memset(sparc64_valid_addr_bitmap, 0, i << 3);
addr = PAGE_OFFSET + kern_base;
last = PAGE_ALIGN(kern_size) + addr;
@@ -1941,15 +1942,19 @@ void __init mem_init(void)
addr += PAGE_SIZE;
}
- setup_valid_addr_bitmap_from_pavail();
+ setup_valid_addr_bitmap_from_pavail(sparc64_valid_addr_bitmap);
+ patch_tlb_miss_handler_bitmap();
high_memory = __va(last_valid_pfn << PAGE_SHIFT);
#ifdef CONFIG_NEED_MULTIPLE_NODES
- for_each_online_node(i) {
- if (NODE_DATA(i)->node_spanned_pages != 0) {
- totalram_pages +=
- free_all_bootmem_node(NODE_DATA(i));
+ {
+ int i;
+ for_each_online_node(i) {
+ if (NODE_DATA(i)->node_spanned_pages != 0) {
+ totalram_pages +=
+ free_all_bootmem_node(NODE_DATA(i));
+ }
}
}
#else
@@ -5,10 +5,13 @@
* marked non-static so that assembler code can get at them.
*/
-#define MAX_PHYS_ADDRESS (1UL << 42UL)
-#define KPTE_BITMAP_CHUNK_SZ (256UL * 1024UL * 1024UL)
+#define MAX_PHYS_ADDRESS (1UL << 41UL)
+#define KPTE_BITMAP_CHUNK_SZ (256UL * 1024UL * 1024UL)
#define KPTE_BITMAP_BYTES \
((MAX_PHYS_ADDRESS / KPTE_BITMAP_CHUNK_SZ) / 8)
+#define VALID_ADDR_BITMAP_CHUNK_SZ (4UL * 1024UL * 1024UL)
+#define VALID_ADDR_BITMAP_BYTES \
+ ((MAX_PHYS_ADDRESS / VALID_ADDR_BITMAP_CHUNK_SZ) / 8)
extern unsigned long kern_linear_pte_xor[2];
extern unsigned long kpte_linear_bitmap[KPTE_BITMAP_BYTES / sizeof(unsigned long)];