|
Lines 1-3
Link Here
|
|
|
1 |
#define DBG KERN_DEBUG "* pcpu debug:" |
| 1 |
/* |
2 |
/* |
| 2 |
* linux/mm/percpu.c - percpu memory allocator |
3 |
* linux/mm/percpu.c - percpu memory allocator |
| 3 |
* |
4 |
* |
|
Lines 719-724
static int pcpu_alloc_pages(struct pcpu_
Link Here
|
| 719 |
struct page **pagep = &pages[pcpu_page_idx(cpu, i)]; |
720 |
struct page **pagep = &pages[pcpu_page_idx(cpu, i)]; |
| 720 |
|
721 |
|
| 721 |
*pagep = alloc_pages_node(cpu_to_node(cpu), gfp, 0); |
722 |
*pagep = alloc_pages_node(cpu_to_node(cpu), gfp, 0); |
|
|
723 |
printk(DBG "pcpu_alloc_pages: pagep=%p, *pagep=%p, cpu=%d, i=%d, phys=%p\n", pagep, *pagep, cpu, i, page_to_phys(*pagep)); |
| 724 |
|
| 722 |
if (!*pagep) { |
725 |
if (!*pagep) { |
| 723 |
pcpu_free_pages(chunk, pages, populated, |
726 |
pcpu_free_pages(chunk, pages, populated, |
| 724 |
page_start, page_end); |
727 |
page_start, page_end); |
|
Lines 852-860
static int pcpu_map_pages(struct pcpu_ch
Link Here
|
| 852 |
|
855 |
|
| 853 |
/* mapping successful, link chunk and mark populated */ |
856 |
/* mapping successful, link chunk and mark populated */ |
| 854 |
for (i = page_start; i < page_end; i++) { |
857 |
for (i = page_start; i < page_end; i++) { |
| 855 |
for_each_possible_cpu(cpu) |
858 |
struct page *p; |
| 856 |
pcpu_set_page_chunk(pages[pcpu_page_idx(cpu, i)], |
859 |
for_each_possible_cpu(cpu) { |
|
|
860 |
pcpu_set_page_chunk(p = pages[pcpu_page_idx(cpu, i)], |
| 857 |
chunk); |
861 |
chunk); |
|
|
862 |
printk(DBG "pcpu_map_pages: page=%p, cpu=%d, i=%d\n", p, cpu, i); |
| 863 |
} |
| 858 |
__set_bit(i, populated); |
864 |
__set_bit(i, populated); |
| 859 |
} |
865 |
} |
| 860 |
|
866 |
|
|
Lines 1000-1007
static int pcpu_populate_chunk(struct pc
Link Here
|
| 1000 |
/* commit new bitmap */ |
1006 |
/* commit new bitmap */ |
| 1001 |
bitmap_copy(chunk->populated, populated, pcpu_unit_pages); |
1007 |
bitmap_copy(chunk->populated, populated, pcpu_unit_pages); |
| 1002 |
clear: |
1008 |
clear: |
| 1003 |
for_each_possible_cpu(cpu) |
1009 |
for_each_possible_cpu(cpu) { |
|
|
1010 |
printk(DBG "going to memset: chunk=%p, cpu=%d, off=%d, size=%d, addr=%p\n", |
| 1011 |
chunk, cpu, off, size, (void*)pcpu_chunk_addr(chunk, cpu, 0)); |
| 1004 |
memset((void *)pcpu_chunk_addr(chunk, cpu, 0) + off, 0, size); |
1012 |
memset((void *)pcpu_chunk_addr(chunk, cpu, 0) + off, 0, size); |
|
|
1013 |
} |
| 1005 |
return 0; |
1014 |
return 0; |
| 1006 |
|
1015 |
|
| 1007 |
err_unmap: |
1016 |
err_unmap: |
|
Lines 1080-1085
static void *pcpu_alloc(size_t size, siz
Link Here
|
| 1080 |
"percpu allocation\n", size, align); |
1089 |
"percpu allocation\n", size, align); |
| 1081 |
return NULL; |
1090 |
return NULL; |
| 1082 |
} |
1091 |
} |
|
|
1092 |
printk(DBG "pcpu_alloc(size=%d, align=%d, reserved=%d\n", size, align, reserved); |
| 1083 |
|
1093 |
|
| 1084 |
mutex_lock(&pcpu_alloc_mutex); |
1094 |
mutex_lock(&pcpu_alloc_mutex); |
| 1085 |
spin_lock_irqsave(&pcpu_lock, flags); |
1095 |
spin_lock_irqsave(&pcpu_lock, flags); |
|
Lines 1114-1123
restart:
Link Here
|
| 1114 |
/* search through normal chunks */ |
1124 |
/* search through normal chunks */ |
| 1115 |
for (slot = pcpu_size_to_slot(size); slot < pcpu_nr_slots; slot++) { |
1125 |
for (slot = pcpu_size_to_slot(size); slot < pcpu_nr_slots; slot++) { |
| 1116 |
list_for_each_entry(chunk, &pcpu_slot[slot], list) { |
1126 |
list_for_each_entry(chunk, &pcpu_slot[slot], list) { |
| 1117 |
if (size > chunk->contig_hint) |
1127 |
if (size > chunk->contig_hint) { |
|
|
1128 |
printk(DBG "chunk %p, contig_hint = %d < %d\n", chunk, chunk->contig_hint, size); |
| 1118 |
continue; |
1129 |
continue; |
|
|
1130 |
} |
| 1119 |
|
1131 |
|
| 1120 |
new_alloc = pcpu_need_to_extend(chunk); |
1132 |
new_alloc = pcpu_need_to_extend(chunk); |
|
|
1133 |
printk(DBG "chunk=%p, new_alloc=%d, chunk->base_addr=%p\n", chunk, new_alloc, chunk->base_addr); |
| 1121 |
if (new_alloc) { |
1134 |
if (new_alloc) { |
| 1122 |
spin_unlock_irqrestore(&pcpu_lock, flags); |
1135 |
spin_unlock_irqrestore(&pcpu_lock, flags); |
| 1123 |
if (pcpu_extend_area_map(chunk, |
1136 |
if (pcpu_extend_area_map(chunk, |
|
Lines 1134-1139
restart:
Link Here
|
| 1134 |
} |
1147 |
} |
| 1135 |
|
1148 |
|
| 1136 |
off = pcpu_alloc_area(chunk, size, align); |
1149 |
off = pcpu_alloc_area(chunk, size, align); |
|
|
1150 |
printk(DBG "off=%d\n", off); |
| 1151 |
|
| 1137 |
if (off >= 0) |
1152 |
if (off >= 0) |
| 1138 |
goto area_found; |
1153 |
goto area_found; |
| 1139 |
} |
1154 |
} |
|
Lines 1147-1153
restart:
Link Here
|
| 1147 |
err = "failed to allocate new chunk"; |
1162 |
err = "failed to allocate new chunk"; |
| 1148 |
goto fail_unlock_mutex; |
1163 |
goto fail_unlock_mutex; |
| 1149 |
} |
1164 |
} |
| 1150 |
|
1165 |
printk(DBG "allocated chunk: %p, base=%p\n", chunk, chunk->base_addr); |
| 1151 |
spin_lock_irqsave(&pcpu_lock, flags); |
1166 |
spin_lock_irqsave(&pcpu_lock, flags); |
| 1152 |
pcpu_chunk_relocate(chunk, -1); |
1167 |
pcpu_chunk_relocate(chunk, -1); |
| 1153 |
goto restart; |
1168 |
goto restart; |