xref: /illumos-kvm-cmd/exec.c (revision 7bdfeb90)
1 /*
2  *  virtual page mapping and translated block handling
3  *
4  *  Copyright (c) 2003 Fabrice Bellard
5  *
6  *  Portions Copyright 2011 Joyent, Inc.
7  *
8  * This library is free software; you can redistribute it and/or
9  * modify it under the terms of the GNU Lesser General Public
10  * License as published by the Free Software Foundation; either
11  * version 2 of the License, or (at your option) any later version.
12  *
13  * This library is distributed in the hope that it will be useful,
14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
16  * Lesser General Public License for more details.
17  *
18  * You should have received a copy of the GNU Lesser General Public
19  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
20  */
21 #include "config.h"
22 #ifdef _WIN32
23 #include <windows.h>
24 #else
25 #include <sys/types.h>
26 #include <sys/mman.h>
27 #endif
28 
29 #include "qemu-common.h"
30 #include "cpu.h"
31 #include "exec-all.h"
32 #include "cache-utils.h"
33 
34 #if !defined(TARGET_IA64)
35 #include "tcg.h"
36 #endif
37 #include "qemu-kvm.h"
38 
39 #include "hw/hw.h"
40 #include "hw/qdev.h"
41 #include "osdep.h"
42 #include "kvm.h"
43 #include "qemu-timer.h"
44 #if defined(CONFIG_USER_ONLY)
45 #include <qemu.h>
46 #include <signal.h>
47 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
48 #include <sys/param.h>
49 #if __FreeBSD_version >= 700104
50 #define HAVE_KINFO_GETVMMAP
51 #define sigqueue sigqueue_freebsd  /* avoid redefinition */
52 #include <sys/time.h>
53 #include <sys/proc.h>
54 #include <machine/profile.h>
55 #define _KERNEL
56 #include <sys/user.h>
57 #undef _KERNEL
58 #undef sigqueue
59 #include <libutil.h>
60 #endif
61 #endif
62 #endif
63 
64 //#define DEBUG_TB_INVALIDATE
65 //#define DEBUG_FLUSH
66 //#define DEBUG_TLB
67 //#define DEBUG_UNASSIGNED
68 
69 /* make various TB consistency checks */
70 //#define DEBUG_TB_CHECK
71 //#define DEBUG_TLB_CHECK
72 
73 //#define DEBUG_IOPORT
74 //#define DEBUG_SUBPAGE
75 
76 #if !defined(CONFIG_USER_ONLY)
77 /* TB consistency checks only implemented for usermode emulation.  */
78 #undef DEBUG_TB_CHECK
79 #endif
80 
81 #define SMC_BITMAP_USE_THRESHOLD 10
82 
83 static TranslationBlock *tbs;
84 static int code_gen_max_blocks;
85 TranslationBlock *tb_phys_hash[CODE_GEN_PHYS_HASH_SIZE];
86 static int nb_tbs;
87 /* any access to the tbs or the page table must use this lock */
88 spinlock_t tb_lock = SPIN_LOCK_UNLOCKED;
89 
90 #if defined(__arm__) || defined(__sparc_v9__)
91 /* The prologue must be reachable with a direct jump. ARM and Sparc64
92  have limited branch ranges (possibly also PPC) so place it in a
93  section close to code segment. */
94 #define code_gen_section                                \
95     __attribute__((__section__(".gen_code")))           \
96     __attribute__((aligned (32)))
97 #elif defined(_WIN32)
98 /* Maximum alignment for Win32 is 16. */
99 #define code_gen_section                                \
100     __attribute__((aligned (16)))
101 #else
102 #define code_gen_section                                \
103     __attribute__((aligned (32)))
104 #endif
105 
106 uint8_t code_gen_prologue[1024] code_gen_section;
107 static uint8_t *code_gen_buffer;
108 static unsigned long code_gen_buffer_size;
109 /* threshold to flush the translated code buffer */
110 static unsigned long code_gen_buffer_max_size;
111 static uint8_t *code_gen_ptr;
112 
113 #if !defined(CONFIG_USER_ONLY)
114 int phys_ram_fd;
115 static int in_migration;
116 
117 RAMList ram_list = { .blocks = QLIST_HEAD_INITIALIZER(ram_list) };
118 #endif
119 
120 CPUState *first_cpu;
121 /* current CPU in the current thread. It is only valid inside
122    cpu_exec() */
123 CPUState *cpu_single_env;
124 /* 0 = Do not count executed instructions.
125    1 = Precise instruction counting.
126    2 = Adaptive rate instruction counting.  */
127 int use_icount = 0;
128 /* Current instruction counter.  While executing translated code this may
129    include some instructions that have not yet been executed.  */
130 int64_t qemu_icount;
131 
132 typedef struct PageDesc {
133     /* list of TBs intersecting this ram page */
134     TranslationBlock *first_tb;
135     /* in order to optimize self modifying code, we count the number
136        of lookups we do to a given page to use a bitmap */
137     unsigned int code_write_count;
138     uint8_t *code_bitmap;
139 #if defined(CONFIG_USER_ONLY)
140     unsigned long flags;
141 #endif
142 } PageDesc;
143 
144 /* In system mode we want L1_MAP to be based on ram offsets,
145    while in user mode we want it to be based on virtual addresses.  */
146 #if !defined(CONFIG_USER_ONLY)
147 #if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS
148 # define L1_MAP_ADDR_SPACE_BITS  HOST_LONG_BITS
149 #else
150 # define L1_MAP_ADDR_SPACE_BITS  TARGET_PHYS_ADDR_SPACE_BITS
151 #endif
152 #else
153 # define L1_MAP_ADDR_SPACE_BITS  TARGET_VIRT_ADDR_SPACE_BITS
154 #endif
155 
156 /* Size of the L2 (and L3, etc) page tables.  */
157 #define L2_BITS 10
158 #define L2_SIZE (1 << L2_BITS)
159 
160 /* The bits remaining after N lower levels of page tables.  */
161 #define P_L1_BITS_REM \
162     ((TARGET_PHYS_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % L2_BITS)
163 #define V_L1_BITS_REM \
164     ((L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % L2_BITS)
165 
166 /* Size of the L1 page table.  Avoid silly small sizes.  */
167 #if P_L1_BITS_REM < 4
168 #define P_L1_BITS  (P_L1_BITS_REM + L2_BITS)
169 #else
170 #define P_L1_BITS  P_L1_BITS_REM
171 #endif
172 
173 #if V_L1_BITS_REM < 4
174 #define V_L1_BITS  (V_L1_BITS_REM + L2_BITS)
175 #else
176 #define V_L1_BITS  V_L1_BITS_REM
177 #endif
178 
179 #define P_L1_SIZE  ((target_phys_addr_t)1 << P_L1_BITS)
180 #define V_L1_SIZE  ((target_ulong)1 << V_L1_BITS)
181 
182 #define P_L1_SHIFT (TARGET_PHYS_ADDR_SPACE_BITS - TARGET_PAGE_BITS - P_L1_BITS)
183 #define V_L1_SHIFT (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - V_L1_BITS)
184 
185 unsigned long qemu_real_host_page_size;
186 unsigned long qemu_host_page_bits;
187 unsigned long qemu_host_page_size;
188 unsigned long qemu_host_page_mask;
189 
190 /* This is a multi-level map on the virtual address space.
191    The bottom level has pointers to PageDesc.  */
192 static void *l1_map[V_L1_SIZE];
193 
194 #if !defined(CONFIG_USER_ONLY)
195 typedef struct PhysPageDesc {
196     /* offset in host memory of the page + io_index in the low bits */
197     ram_addr_t phys_offset;
198     ram_addr_t region_offset;
199 } PhysPageDesc;
200 
201 /* This is a multi-level map on the physical address space.
202    The bottom level has pointers to PhysPageDesc.  */
203 static void *l1_phys_map[P_L1_SIZE];
204 
205 static void io_mem_init(void);
206 
207 /* io memory support */
208 CPUWriteMemoryFunc *io_mem_write[IO_MEM_NB_ENTRIES][4];
209 CPUReadMemoryFunc *io_mem_read[IO_MEM_NB_ENTRIES][4];
210 void *io_mem_opaque[IO_MEM_NB_ENTRIES];
211 static char io_mem_used[IO_MEM_NB_ENTRIES];
212 static int io_mem_watch;
213 #endif
214 
215 /* log support */
216 #ifdef WIN32
217 static const char *logfilename = "qemu.log";
218 #else
219 static const char *logfilename = "/tmp/qemu.log";
220 #endif
221 FILE *logfile;
222 int loglevel;
223 static int log_append = 0;
224 
225 /* statistics */
226 #if !defined(CONFIG_USER_ONLY)
227 static int tlb_flush_count;
228 #endif
229 static int tb_flush_count;
230 static int tb_phys_invalidate_count;
231 
232 #ifdef _WIN32
map_exec(void * addr,long size)233 static void map_exec(void *addr, long size)
234 {
235     DWORD old_protect;
236     VirtualProtect(addr, size,
237                    PAGE_EXECUTE_READWRITE, &old_protect);
238 
239 }
240 #else
map_exec(void * addr,long size)241 static void map_exec(void *addr, long size)
242 {
243     unsigned long start, end, page_size;
244 
245     page_size = getpagesize();
246     start = (unsigned long)addr;
247     start &= ~(page_size - 1);
248 
249     end = (unsigned long)addr + size;
250     end += page_size - 1;
251     end &= ~(page_size - 1);
252 
253     mprotect((void *)start, end - start,
254              PROT_READ | PROT_WRITE | PROT_EXEC);
255 }
256 #endif
257 
page_init(void)258 static void page_init(void)
259 {
260     /* NOTE: we can always suppose that qemu_host_page_size >=
261        TARGET_PAGE_SIZE */
262 #ifdef _WIN32
263     {
264         SYSTEM_INFO system_info;
265 
266         GetSystemInfo(&system_info);
267         qemu_real_host_page_size = system_info.dwPageSize;
268     }
269 #else
270     qemu_real_host_page_size = getpagesize();
271 #endif
272     if (qemu_host_page_size == 0)
273         qemu_host_page_size = qemu_real_host_page_size;
274     if (qemu_host_page_size < TARGET_PAGE_SIZE)
275         qemu_host_page_size = TARGET_PAGE_SIZE;
276     qemu_host_page_bits = 0;
277     while ((1 << qemu_host_page_bits) < qemu_host_page_size)
278         qemu_host_page_bits++;
279     qemu_host_page_mask = ~(qemu_host_page_size - 1);
280 
281 #if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY)
282     {
283 #ifdef HAVE_KINFO_GETVMMAP
284         struct kinfo_vmentry *freep;
285         int i, cnt;
286 
287         freep = kinfo_getvmmap(getpid(), &cnt);
288         if (freep) {
289             mmap_lock();
290             for (i = 0; i < cnt; i++) {
291                 unsigned long startaddr, endaddr;
292 
293                 startaddr = freep[i].kve_start;
294                 endaddr = freep[i].kve_end;
295                 if (h2g_valid(startaddr)) {
296                     startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
297 
298                     if (h2g_valid(endaddr)) {
299                         endaddr = h2g(endaddr);
300                         page_set_flags(startaddr, endaddr, PAGE_RESERVED);
301                     } else {
302 #if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS
303                         endaddr = ~0ul;
304                         page_set_flags(startaddr, endaddr, PAGE_RESERVED);
305 #endif
306                     }
307                 }
308             }
309             free(freep);
310             mmap_unlock();
311         }
312 #else
313         FILE *f;
314 
315         last_brk = (unsigned long)sbrk(0);
316 
317         f = fopen("/compat/linux/proc/self/maps", "r");
318         if (f) {
319             mmap_lock();
320 
321             do {
322                 unsigned long startaddr, endaddr;
323                 int n;
324 
325                 n = fscanf (f, "%lx-%lx %*[^\n]\n", &startaddr, &endaddr);
326 
327                 if (n == 2 && h2g_valid(startaddr)) {
328                     startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
329 
330                     if (h2g_valid(endaddr)) {
331                         endaddr = h2g(endaddr);
332                     } else {
333                         endaddr = ~0ul;
334                     }
335                     page_set_flags(startaddr, endaddr, PAGE_RESERVED);
336                 }
337             } while (!feof(f));
338 
339             fclose(f);
340             mmap_unlock();
341         }
342 #endif
343     }
344 #endif
345 }
346 
page_find_alloc(tb_page_addr_t index,int alloc)347 static PageDesc *page_find_alloc(tb_page_addr_t index, int alloc)
348 {
349     PageDesc *pd;
350     void **lp;
351     int i;
352 
353 #if defined(CONFIG_USER_ONLY)
354     /* We can't use qemu_malloc because it may recurse into a locked mutex. */
355 # define ALLOC(P, SIZE)                                 \
356     do {                                                \
357         P = mmap(NULL, SIZE, PROT_READ | PROT_WRITE,    \
358                  MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);   \
359     } while (0)
360 #else
361 # define ALLOC(P, SIZE) \
362     do { P = qemu_mallocz(SIZE); } while (0)
363 #endif
364 
365     /* Level 1.  Always allocated.  */
366     lp = l1_map + ((index >> V_L1_SHIFT) & (V_L1_SIZE - 1));
367 
368     /* Level 2..N-1.  */
369     for (i = V_L1_SHIFT / L2_BITS - 1; i > 0; i--) {
370         void **p = *lp;
371 
372         if (p == NULL) {
373             if (!alloc) {
374                 return NULL;
375             }
376             ALLOC(p, sizeof(void *) * L2_SIZE);
377             *lp = p;
378         }
379 
380         lp = p + ((index >> (i * L2_BITS)) & (L2_SIZE - 1));
381     }
382 
383     pd = *lp;
384     if (pd == NULL) {
385         if (!alloc) {
386             return NULL;
387         }
388         ALLOC(pd, sizeof(PageDesc) * L2_SIZE);
389         *lp = pd;
390     }
391 
392 #undef ALLOC
393 
394     return pd + (index & (L2_SIZE - 1));
395 }
396 
page_find(tb_page_addr_t index)397 static inline PageDesc *page_find(tb_page_addr_t index)
398 {
399     return page_find_alloc(index, 0);
400 }
401 
402 #if !defined(CONFIG_USER_ONLY)
phys_page_find_alloc(target_phys_addr_t index,int alloc)403 static PhysPageDesc *phys_page_find_alloc(target_phys_addr_t index, int alloc)
404 {
405     PhysPageDesc *pd;
406     void **lp;
407     int i;
408 
409     /* Level 1.  Always allocated.  */
410     lp = l1_phys_map + ((index >> P_L1_SHIFT) & (P_L1_SIZE - 1));
411 
412     /* Level 2..N-1.  */
413     for (i = P_L1_SHIFT / L2_BITS - 1; i > 0; i--) {
414         void **p = *lp;
415         if (p == NULL) {
416             if (!alloc) {
417                 return NULL;
418             }
419             *lp = p = qemu_mallocz(sizeof(void *) * L2_SIZE);
420         }
421         lp = p + ((index >> (i * L2_BITS)) & (L2_SIZE - 1));
422     }
423 
424     pd = *lp;
425     if (pd == NULL) {
426         int i;
427 
428         if (!alloc) {
429             return NULL;
430         }
431 
432         *lp = pd = qemu_malloc(sizeof(PhysPageDesc) * L2_SIZE);
433 
434         for (i = 0; i < L2_SIZE; i++) {
435             pd[i].phys_offset = IO_MEM_UNASSIGNED;
436             pd[i].region_offset = (index + i) << TARGET_PAGE_BITS;
437         }
438     }
439 
440     return pd + (index & (L2_SIZE - 1));
441 }
442 
phys_page_find(target_phys_addr_t index)443 static inline PhysPageDesc *phys_page_find(target_phys_addr_t index)
444 {
445     return phys_page_find_alloc(index, 0);
446 }
447 
448 static void tlb_protect_code(ram_addr_t ram_addr);
449 static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr,
450                                     target_ulong vaddr);
451 #define mmap_lock() do { } while(0)
452 #define mmap_unlock() do { } while(0)
453 #endif
454 
455 #define DEFAULT_CODE_GEN_BUFFER_SIZE (32 * 1024 * 1024)
456 
457 #if defined(CONFIG_USER_ONLY)
458 /* Currently it is not recommended to allocate big chunks of data in
459    user mode. It will change when a dedicated libc will be used */
460 #define USE_STATIC_CODE_GEN_BUFFER
461 #endif
462 
463 #ifdef USE_STATIC_CODE_GEN_BUFFER
464 static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE]
465                __attribute__((aligned (CODE_GEN_ALIGN)));
466 #endif
467 
code_gen_alloc(unsigned long tb_size)468 static void code_gen_alloc(unsigned long tb_size)
469 {
470     if (kvm_enabled())
471         return;
472 
473 #ifdef USE_STATIC_CODE_GEN_BUFFER
474     code_gen_buffer = static_code_gen_buffer;
475     code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
476     map_exec(code_gen_buffer, code_gen_buffer_size);
477 #else
478     code_gen_buffer_size = tb_size;
479     if (code_gen_buffer_size == 0) {
480 #if defined(CONFIG_USER_ONLY)
481         /* in user mode, phys_ram_size is not meaningful */
482         code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
483 #else
484         /* XXX: needs adjustments */
485         code_gen_buffer_size = (unsigned long)(ram_size / 4);
486 #endif
487     }
488     if (code_gen_buffer_size < MIN_CODE_GEN_BUFFER_SIZE)
489         code_gen_buffer_size = MIN_CODE_GEN_BUFFER_SIZE;
490     /* The code gen buffer location may have constraints depending on
491        the host cpu and OS */
492 #if defined(__linux__)
493     {
494         int flags;
495         void *start = NULL;
496 
497         flags = MAP_PRIVATE | MAP_ANONYMOUS;
498 #if defined(__x86_64__)
499         flags |= MAP_32BIT;
500         /* Cannot map more than that */
501         if (code_gen_buffer_size > (800 * 1024 * 1024))
502             code_gen_buffer_size = (800 * 1024 * 1024);
503 #elif defined(__sparc_v9__)
504         // Map the buffer below 2G, so we can use direct calls and branches
505         flags |= MAP_FIXED;
506         start = (void *) 0x60000000UL;
507         if (code_gen_buffer_size > (512 * 1024 * 1024))
508             code_gen_buffer_size = (512 * 1024 * 1024);
509 #elif defined(__arm__)
510         /* Map the buffer below 32M, so we can use direct calls and branches */
511         flags |= MAP_FIXED;
512         start = (void *) 0x01000000UL;
513         if (code_gen_buffer_size > 16 * 1024 * 1024)
514             code_gen_buffer_size = 16 * 1024 * 1024;
515 #elif defined(__s390x__)
516         /* Map the buffer so that we can use direct calls and branches.  */
517         /* We have a +- 4GB range on the branches; leave some slop.  */
518         if (code_gen_buffer_size > (3ul * 1024 * 1024 * 1024)) {
519             code_gen_buffer_size = 3ul * 1024 * 1024 * 1024;
520         }
521         start = (void *)0x90000000UL;
522 #endif
523         code_gen_buffer = mmap(start, code_gen_buffer_size,
524                                PROT_WRITE | PROT_READ | PROT_EXEC,
525                                flags, -1, 0);
526         if (code_gen_buffer == MAP_FAILED) {
527             fprintf(stderr, "Could not allocate dynamic translator buffer\n");
528             exit(1);
529         }
530     }
531 #elif defined(__FreeBSD__) || defined(__FreeBSD_kernel__) \
532     || defined(__DragonFly__) || defined(__OpenBSD__)
533     {
534         int flags;
535         void *addr = NULL;
536         flags = MAP_PRIVATE | MAP_ANONYMOUS;
537 #if defined(__x86_64__)
538         /* FreeBSD doesn't have MAP_32BIT, use MAP_FIXED and assume
539          * 0x40000000 is free */
540         flags |= MAP_FIXED;
541         addr = (void *)0x40000000;
542         /* Cannot map more than that */
543         if (code_gen_buffer_size > (800 * 1024 * 1024))
544             code_gen_buffer_size = (800 * 1024 * 1024);
545 #elif defined(__sparc_v9__)
546         // Map the buffer below 2G, so we can use direct calls and branches
547         flags |= MAP_FIXED;
548         addr = (void *) 0x60000000UL;
549         if (code_gen_buffer_size > (512 * 1024 * 1024)) {
550             code_gen_buffer_size = (512 * 1024 * 1024);
551         }
552 #endif
553         code_gen_buffer = mmap(addr, code_gen_buffer_size,
554                                PROT_WRITE | PROT_READ | PROT_EXEC,
555                                flags, -1, 0);
556         if (code_gen_buffer == MAP_FAILED) {
557             fprintf(stderr, "Could not allocate dynamic translator buffer\n");
558             exit(1);
559         }
560     }
561 #else
562     code_gen_buffer = qemu_malloc(code_gen_buffer_size);
563     map_exec(code_gen_buffer, code_gen_buffer_size);
564 #endif
565 #endif /* !USE_STATIC_CODE_GEN_BUFFER */
566     map_exec(code_gen_prologue, sizeof(code_gen_prologue));
567     code_gen_buffer_max_size = code_gen_buffer_size -
568         (TCG_MAX_OP_SIZE * OPC_MAX_SIZE);
569     code_gen_max_blocks = code_gen_buffer_size / CODE_GEN_AVG_BLOCK_SIZE;
570     tbs = qemu_malloc(code_gen_max_blocks * sizeof(TranslationBlock));
571 }
572 
573 /* Must be called before using the QEMU cpus. 'tb_size' is the size
574    (in bytes) allocated to the translation buffer. Zero means default
575    size. */
cpu_exec_init_all(unsigned long tb_size)576 void cpu_exec_init_all(unsigned long tb_size)
577 {
578     cpu_gen_init();
579     code_gen_alloc(tb_size);
580     code_gen_ptr = code_gen_buffer;
581     page_init();
582 #if !defined(CONFIG_USER_ONLY)
583     io_mem_init();
584 #endif
585 #if !defined(CONFIG_USER_ONLY) || !defined(CONFIG_USE_GUEST_BASE)
586     /* There's no guest base to take into account, so go ahead and
587        initialize the prologue now.  */
588     tcg_prologue_init(&tcg_ctx);
589 #endif
590 }
591 
592 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
593 
cpu_common_post_load(void * opaque,int version_id)594 static int cpu_common_post_load(void *opaque, int version_id)
595 {
596     CPUState *env = opaque;
597 
598     /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
599        version_id is increased. */
600     env->interrupt_request &= ~0x01;
601     tlb_flush(env, 1);
602 
603     return 0;
604 }
605 
606 static const VMStateDescription vmstate_cpu_common = {
607     .name = "cpu_common",
608     .version_id = 1,
609     .minimum_version_id = 1,
610     .minimum_version_id_old = 1,
611     .post_load = cpu_common_post_load,
612     .fields      = (VMStateField []) {
613         VMSTATE_UINT32(halted, CPUState),
614         VMSTATE_UINT32(interrupt_request, CPUState),
615         VMSTATE_END_OF_LIST()
616     }
617 };
618 #endif
619 
qemu_get_cpu(int cpu)620 CPUState *qemu_get_cpu(int cpu)
621 {
622     CPUState *env = first_cpu;
623 
624     while (env) {
625         if (env->cpu_index == cpu)
626             break;
627         env = env->next_cpu;
628     }
629 
630     return env;
631 }
632 
cpu_exec_init(CPUState * env)633 void cpu_exec_init(CPUState *env)
634 {
635     CPUState **penv;
636     int cpu_index;
637 
638 #if defined(CONFIG_USER_ONLY)
639     cpu_list_lock();
640 #endif
641     env->next_cpu = NULL;
642     penv = &first_cpu;
643     cpu_index = 0;
644     while (*penv != NULL) {
645         penv = &(*penv)->next_cpu;
646         cpu_index++;
647     }
648     env->cpu_index = cpu_index;
649     env->numa_node = 0;
650     QTAILQ_INIT(&env->breakpoints);
651     QTAILQ_INIT(&env->watchpoints);
652 #ifdef __WIN32
653     env->thread_id = GetCurrentProcessId();
654 #else
655     env->thread_id = getpid();
656 #endif
657     *penv = env;
658 #if defined(CONFIG_USER_ONLY)
659     cpu_list_unlock();
660 #endif
661 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
662     vmstate_register(NULL, cpu_index, &vmstate_cpu_common, env);
663     register_savevm(NULL, "cpu", cpu_index, CPU_SAVE_VERSION,
664                     cpu_save, cpu_load, env);
665 #endif
666 }
667 
invalidate_page_bitmap(PageDesc * p)668 static inline void invalidate_page_bitmap(PageDesc *p)
669 {
670     if (p->code_bitmap) {
671         qemu_free(p->code_bitmap);
672         p->code_bitmap = NULL;
673     }
674     p->code_write_count = 0;
675 }
676 
677 /* Set to NULL all the 'first_tb' fields in all PageDescs. */
678 
page_flush_tb_1(int level,void ** lp)679 static void page_flush_tb_1 (int level, void **lp)
680 {
681     int i;
682 
683     if (*lp == NULL) {
684         return;
685     }
686     if (level == 0) {
687         PageDesc *pd = *lp;
688         for (i = 0; i < L2_SIZE; ++i) {
689             pd[i].first_tb = NULL;
690             invalidate_page_bitmap(pd + i);
691         }
692     } else {
693         void **pp = *lp;
694         for (i = 0; i < L2_SIZE; ++i) {
695             page_flush_tb_1 (level - 1, pp + i);
696         }
697     }
698 }
699 
page_flush_tb(void)700 static void page_flush_tb(void)
701 {
702     int i;
703     for (i = 0; i < V_L1_SIZE; i++) {
704         page_flush_tb_1(V_L1_SHIFT / L2_BITS - 1, l1_map + i);
705     }
706 }
707 
708 /* flush all the translation blocks */
709 /* XXX: tb_flush is currently not thread safe */
tb_flush(CPUState * env1)710 void tb_flush(CPUState *env1)
711 {
712     CPUState *env;
713 #if defined(DEBUG_FLUSH)
714     printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
715            (unsigned long)(code_gen_ptr - code_gen_buffer),
716            nb_tbs, nb_tbs > 0 ?
717            ((unsigned long)(code_gen_ptr - code_gen_buffer)) / nb_tbs : 0);
718 #endif
719     if ((unsigned long)(code_gen_ptr - code_gen_buffer) > code_gen_buffer_size)
720         cpu_abort(env1, "Internal error: code buffer overflow\n");
721 
722     nb_tbs = 0;
723 
724     for(env = first_cpu; env != NULL; env = env->next_cpu) {
725         memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
726     }
727 
728     memset (tb_phys_hash, 0, CODE_GEN_PHYS_HASH_SIZE * sizeof (void *));
729     page_flush_tb();
730 
731     code_gen_ptr = code_gen_buffer;
732     /* XXX: flush processor icache at this point if cache flush is
733        expensive */
734     tb_flush_count++;
735 }
736 
737 #ifdef DEBUG_TB_CHECK
738 
tb_invalidate_check(target_ulong address)739 static void tb_invalidate_check(target_ulong address)
740 {
741     TranslationBlock *tb;
742     int i;
743     address &= TARGET_PAGE_MASK;
744     for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
745         for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
746             if (!(address + TARGET_PAGE_SIZE <= tb->pc ||
747                   address >= tb->pc + tb->size)) {
748                 printf("ERROR invalidate: address=" TARGET_FMT_lx
749                        " PC=%08lx size=%04x\n",
750                        address, (long)tb->pc, tb->size);
751             }
752         }
753     }
754 }
755 
756 /* verify that all the pages have correct rights for code */
tb_page_check(void)757 static void tb_page_check(void)
758 {
759     TranslationBlock *tb;
760     int i, flags1, flags2;
761 
762     for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
763         for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
764             flags1 = page_get_flags(tb->pc);
765             flags2 = page_get_flags(tb->pc + tb->size - 1);
766             if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) {
767                 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
768                        (long)tb->pc, tb->size, flags1, flags2);
769             }
770         }
771     }
772 }
773 
774 #endif
775 
776 /* invalidate one TB */
tb_remove(TranslationBlock ** ptb,TranslationBlock * tb,int next_offset)777 static inline void tb_remove(TranslationBlock **ptb, TranslationBlock *tb,
778                              int next_offset)
779 {
780     TranslationBlock *tb1;
781     for(;;) {
782         tb1 = *ptb;
783         if (tb1 == tb) {
784             *ptb = *(TranslationBlock **)((char *)tb1 + next_offset);
785             break;
786         }
787         ptb = (TranslationBlock **)((char *)tb1 + next_offset);
788     }
789 }
790 
tb_page_remove(TranslationBlock ** ptb,TranslationBlock * tb)791 static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb)
792 {
793     TranslationBlock *tb1;
794     unsigned int n1;
795 
796     for(;;) {
797         tb1 = *ptb;
798         n1 = (long)tb1 & 3;
799         tb1 = (TranslationBlock *)((long)tb1 & ~3);
800         if (tb1 == tb) {
801             *ptb = tb1->page_next[n1];
802             break;
803         }
804         ptb = &tb1->page_next[n1];
805     }
806 }
807 
tb_jmp_remove(TranslationBlock * tb,int n)808 static inline void tb_jmp_remove(TranslationBlock *tb, int n)
809 {
810     TranslationBlock *tb1, **ptb;
811     unsigned int n1;
812 
813     ptb = &tb->jmp_next[n];
814     tb1 = *ptb;
815     if (tb1) {
816         /* find tb(n) in circular list */
817         for(;;) {
818             tb1 = *ptb;
819             n1 = (long)tb1 & 3;
820             tb1 = (TranslationBlock *)((long)tb1 & ~3);
821             if (n1 == n && tb1 == tb)
822                 break;
823             if (n1 == 2) {
824                 ptb = &tb1->jmp_first;
825             } else {
826                 ptb = &tb1->jmp_next[n1];
827             }
828         }
829         /* now we can suppress tb(n) from the list */
830         *ptb = tb->jmp_next[n];
831 
832         tb->jmp_next[n] = NULL;
833     }
834 }
835 
836 /* reset the jump entry 'n' of a TB so that it is not chained to
837    another TB */
tb_reset_jump(TranslationBlock * tb,int n)838 static inline void tb_reset_jump(TranslationBlock *tb, int n)
839 {
840     tb_set_jmp_target(tb, n, (unsigned long)(tb->tc_ptr + tb->tb_next_offset[n]));
841 }
842 
tb_phys_invalidate(TranslationBlock * tb,tb_page_addr_t page_addr)843 void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr)
844 {
845     CPUState *env;
846     PageDesc *p;
847     unsigned int h, n1;
848     tb_page_addr_t phys_pc;
849     TranslationBlock *tb1, *tb2;
850 
851     /* remove the TB from the hash list */
852     phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
853     h = tb_phys_hash_func(phys_pc);
854     tb_remove(&tb_phys_hash[h], tb,
855               offsetof(TranslationBlock, phys_hash_next));
856 
857     /* remove the TB from the page list */
858     if (tb->page_addr[0] != page_addr) {
859         p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
860         tb_page_remove(&p->first_tb, tb);
861         invalidate_page_bitmap(p);
862     }
863     if (tb->page_addr[1] != -1 && tb->page_addr[1] != page_addr) {
864         p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
865         tb_page_remove(&p->first_tb, tb);
866         invalidate_page_bitmap(p);
867     }
868 
869     tb_invalidated_flag = 1;
870 
871     /* remove the TB from the hash list */
872     h = tb_jmp_cache_hash_func(tb->pc);
873     for(env = first_cpu; env != NULL; env = env->next_cpu) {
874         if (env->tb_jmp_cache[h] == tb)
875             env->tb_jmp_cache[h] = NULL;
876     }
877 
878     /* suppress this TB from the two jump lists */
879     tb_jmp_remove(tb, 0);
880     tb_jmp_remove(tb, 1);
881 
882     /* suppress any remaining jumps to this TB */
883     tb1 = tb->jmp_first;
884     for(;;) {
885         n1 = (long)tb1 & 3;
886         if (n1 == 2)
887             break;
888         tb1 = (TranslationBlock *)((long)tb1 & ~3);
889         tb2 = tb1->jmp_next[n1];
890         tb_reset_jump(tb1, n1);
891         tb1->jmp_next[n1] = NULL;
892         tb1 = tb2;
893     }
894     tb->jmp_first = (TranslationBlock *)((long)tb | 2); /* fail safe */
895 
896     tb_phys_invalidate_count++;
897 }
898 
set_bits(uint8_t * tab,int start,int len)899 static inline void set_bits(uint8_t *tab, int start, int len)
900 {
901     int end, mask, end1;
902 
903     end = start + len;
904     tab += start >> 3;
905     mask = 0xff << (start & 7);
906     if ((start & ~7) == (end & ~7)) {
907         if (start < end) {
908             mask &= ~(0xff << (end & 7));
909             *tab |= mask;
910         }
911     } else {
912         *tab++ |= mask;
913         start = (start + 8) & ~7;
914         end1 = end & ~7;
915         while (start < end1) {
916             *tab++ = 0xff;
917             start += 8;
918         }
919         if (start < end) {
920             mask = ~(0xff << (end & 7));
921             *tab |= mask;
922         }
923     }
924 }
925 
build_page_bitmap(PageDesc * p)926 static void build_page_bitmap(PageDesc *p)
927 {
928     int n, tb_start, tb_end;
929     TranslationBlock *tb;
930 
931     p->code_bitmap = qemu_mallocz(TARGET_PAGE_SIZE / 8);
932 
933     tb = p->first_tb;
934     while (tb != NULL) {
935         n = (long)tb & 3;
936         tb = (TranslationBlock *)((long)tb & ~3);
937         /* NOTE: this is subtle as a TB may span two physical pages */
938         if (n == 0) {
939             /* NOTE: tb_end may be after the end of the page, but
940                it is not a problem */
941             tb_start = tb->pc & ~TARGET_PAGE_MASK;
942             tb_end = tb_start + tb->size;
943             if (tb_end > TARGET_PAGE_SIZE)
944                 tb_end = TARGET_PAGE_SIZE;
945         } else {
946             tb_start = 0;
947             tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
948         }
949         set_bits(p->code_bitmap, tb_start, tb_end - tb_start);
950         tb = tb->page_next[n];
951     }
952 }
953 
tb_gen_code(CPUState * env,target_ulong pc,target_ulong cs_base,int flags,int cflags)954 TranslationBlock *tb_gen_code(CPUState *env,
955                               target_ulong pc, target_ulong cs_base,
956                               int flags, int cflags)
957 {
958     TranslationBlock *tb;
959     uint8_t *tc_ptr;
960     tb_page_addr_t phys_pc, phys_page2;
961     target_ulong virt_page2;
962     int code_gen_size;
963 
964     phys_pc = get_page_addr_code(env, pc);
965     tb = tb_alloc(pc);
966     if (!tb) {
967         /* flush must be done */
968         tb_flush(env);
969         /* cannot fail at this point */
970         tb = tb_alloc(pc);
971         /* Don't forget to invalidate previous TB info.  */
972         tb_invalidated_flag = 1;
973     }
974     tc_ptr = code_gen_ptr;
975     tb->tc_ptr = tc_ptr;
976     tb->cs_base = cs_base;
977     tb->flags = flags;
978     tb->cflags = cflags;
979     cpu_gen_code(env, tb, &code_gen_size);
980     code_gen_ptr = (void *)(((unsigned long)code_gen_ptr + code_gen_size + CODE_GEN_ALIGN - 1) & ~(CODE_GEN_ALIGN - 1));
981 
982     /* check next page if needed */
983     virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK;
984     phys_page2 = -1;
985     if ((pc & TARGET_PAGE_MASK) != virt_page2) {
986         phys_page2 = get_page_addr_code(env, virt_page2);
987     }
988     tb_link_page(tb, phys_pc, phys_page2);
989     return tb;
990 }
991 
992 /* invalidate all TBs which intersect with the target physical page
993    starting in range [start;end[. NOTE: start and end must refer to
994    the same physical page. 'is_cpu_write_access' should be true if called
995    from a real cpu write access: the virtual CPU will exit the current
996    TB if code is modified inside this TB. */
tb_invalidate_phys_page_range(tb_page_addr_t start,tb_page_addr_t end,int is_cpu_write_access)997 void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end,
998                                    int is_cpu_write_access)
999 {
1000     TranslationBlock *tb, *tb_next, *saved_tb;
1001     CPUState *env = cpu_single_env;
1002     tb_page_addr_t tb_start, tb_end;
1003     PageDesc *p;
1004     int n;
1005 #ifdef TARGET_HAS_PRECISE_SMC
1006     int current_tb_not_found = is_cpu_write_access;
1007     TranslationBlock *current_tb = NULL;
1008     int current_tb_modified = 0;
1009     target_ulong current_pc = 0;
1010     target_ulong current_cs_base = 0;
1011     int current_flags = 0;
1012 #endif /* TARGET_HAS_PRECISE_SMC */
1013 
1014     p = page_find(start >> TARGET_PAGE_BITS);
1015     if (!p)
1016         return;
1017     if (!p->code_bitmap &&
1018         ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD &&
1019         is_cpu_write_access) {
1020         /* build code bitmap */
1021         build_page_bitmap(p);
1022     }
1023 
1024     /* we remove all the TBs in the range [start, end[ */
1025     /* XXX: see if in some cases it could be faster to invalidate all the code */
1026     tb = p->first_tb;
1027     while (tb != NULL) {
1028         n = (long)tb & 3;
1029         tb = (TranslationBlock *)((long)tb & ~3);
1030         tb_next = tb->page_next[n];
1031         /* NOTE: this is subtle as a TB may span two physical pages */
1032         if (n == 0) {
1033             /* NOTE: tb_end may be after the end of the page, but
1034                it is not a problem */
1035             tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
1036             tb_end = tb_start + tb->size;
1037         } else {
1038             tb_start = tb->page_addr[1];
1039             tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
1040         }
1041         if (!(tb_end <= start || tb_start >= end)) {
1042 #ifdef TARGET_HAS_PRECISE_SMC
1043             if (current_tb_not_found) {
1044                 current_tb_not_found = 0;
1045                 current_tb = NULL;
1046                 if (env->mem_io_pc) {
1047                     /* now we have a real cpu fault */
1048                     current_tb = tb_find_pc(env->mem_io_pc);
1049                 }
1050             }
1051             if (current_tb == tb &&
1052                 (current_tb->cflags & CF_COUNT_MASK) != 1) {
1053                 /* If we are modifying the current TB, we must stop
1054                 its execution. We could be more precise by checking
1055                 that the modification is after the current PC, but it
1056                 would require a specialized function to partially
1057                 restore the CPU state */
1058 
1059                 current_tb_modified = 1;
1060                 cpu_restore_state(current_tb, env,
1061                                   env->mem_io_pc, NULL);
1062                 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1063                                      &current_flags);
1064             }
1065 #endif /* TARGET_HAS_PRECISE_SMC */
1066             /* we need to do that to handle the case where a signal
1067                occurs while doing tb_phys_invalidate() */
1068             saved_tb = NULL;
1069             if (env) {
1070                 saved_tb = env->current_tb;
1071                 env->current_tb = NULL;
1072             }
1073             tb_phys_invalidate(tb, -1);
1074             if (env) {
1075                 env->current_tb = saved_tb;
1076                 if (env->interrupt_request && env->current_tb)
1077                     cpu_interrupt(env, env->interrupt_request);
1078             }
1079         }
1080         tb = tb_next;
1081     }
1082 #if !defined(CONFIG_USER_ONLY)
1083     /* if no code remaining, no need to continue to use slow writes */
1084     if (!p->first_tb) {
1085         invalidate_page_bitmap(p);
1086         if (is_cpu_write_access) {
1087             tlb_unprotect_code_phys(env, start, env->mem_io_vaddr);
1088         }
1089     }
1090 #endif
1091 #ifdef TARGET_HAS_PRECISE_SMC
1092     if (current_tb_modified) {
1093         /* we generate a block containing just the instruction
1094            modifying the memory. It will ensure that it cannot modify
1095            itself */
1096         env->current_tb = NULL;
1097         tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
1098         cpu_resume_from_signal(env, NULL);
1099     }
1100 #endif
1101 }
1102 
1103 /* len must be <= 8 and start must be a multiple of len */
tb_invalidate_phys_page_fast(tb_page_addr_t start,int len)1104 static inline void tb_invalidate_phys_page_fast(tb_page_addr_t start, int len)
1105 {
1106     PageDesc *p;
1107     int offset, b;
1108 #if 0
1109     if (1) {
1110         qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1111                   cpu_single_env->mem_io_vaddr, len,
1112                   cpu_single_env->eip,
1113                   cpu_single_env->eip + (long)cpu_single_env->segs[R_CS].base);
1114     }
1115 #endif
1116     p = page_find(start >> TARGET_PAGE_BITS);
1117     if (!p)
1118         return;
1119     if (p->code_bitmap) {
1120         offset = start & ~TARGET_PAGE_MASK;
1121         b = p->code_bitmap[offset >> 3] >> (offset & 7);
1122         if (b & ((1 << len) - 1))
1123             goto do_invalidate;
1124     } else {
1125     do_invalidate:
1126         tb_invalidate_phys_page_range(start, start + len, 1);
1127     }
1128 }
1129 
1130 #if !defined(CONFIG_SOFTMMU)
tb_invalidate_phys_page(tb_page_addr_t addr,unsigned long pc,void * puc)1131 static void tb_invalidate_phys_page(tb_page_addr_t addr,
1132                                     unsigned long pc, void *puc)
1133 {
1134     TranslationBlock *tb;
1135     PageDesc *p;
1136     int n;
1137 #ifdef TARGET_HAS_PRECISE_SMC
1138     TranslationBlock *current_tb = NULL;
1139     CPUState *env = cpu_single_env;
1140     int current_tb_modified = 0;
1141     target_ulong current_pc = 0;
1142     target_ulong current_cs_base = 0;
1143     int current_flags = 0;
1144 #endif
1145 
1146     addr &= TARGET_PAGE_MASK;
1147     p = page_find(addr >> TARGET_PAGE_BITS);
1148     if (!p)
1149         return;
1150     tb = p->first_tb;
1151 #ifdef TARGET_HAS_PRECISE_SMC
1152     if (tb && pc != 0) {
1153         current_tb = tb_find_pc(pc);
1154     }
1155 #endif
1156     while (tb != NULL) {
1157         n = (long)tb & 3;
1158         tb = (TranslationBlock *)((long)tb & ~3);
1159 #ifdef TARGET_HAS_PRECISE_SMC
1160         if (current_tb == tb &&
1161             (current_tb->cflags & CF_COUNT_MASK) != 1) {
1162                 /* If we are modifying the current TB, we must stop
1163                    its execution. We could be more precise by checking
1164                    that the modification is after the current PC, but it
1165                    would require a specialized function to partially
1166                    restore the CPU state */
1167 
1168             current_tb_modified = 1;
1169             cpu_restore_state(current_tb, env, pc, puc);
1170             cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1171                                  &current_flags);
1172         }
1173 #endif /* TARGET_HAS_PRECISE_SMC */
1174         tb_phys_invalidate(tb, addr);
1175         tb = tb->page_next[n];
1176     }
1177     p->first_tb = NULL;
1178 #ifdef TARGET_HAS_PRECISE_SMC
1179     if (current_tb_modified) {
1180         /* we generate a block containing just the instruction
1181            modifying the memory. It will ensure that it cannot modify
1182            itself */
1183         env->current_tb = NULL;
1184         tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
1185         cpu_resume_from_signal(env, puc);
1186     }
1187 #endif
1188 }
1189 #endif
1190 
1191 /* add the tb in the target page and protect it if necessary */
tb_alloc_page(TranslationBlock * tb,unsigned int n,tb_page_addr_t page_addr)1192 static inline void tb_alloc_page(TranslationBlock *tb,
1193                                  unsigned int n, tb_page_addr_t page_addr)
1194 {
1195     PageDesc *p;
1196     TranslationBlock *last_first_tb;
1197 
1198     tb->page_addr[n] = page_addr;
1199     p = page_find_alloc(page_addr >> TARGET_PAGE_BITS, 1);
1200     tb->page_next[n] = p->first_tb;
1201     last_first_tb = p->first_tb;
1202     p->first_tb = (TranslationBlock *)((long)tb | n);
1203     invalidate_page_bitmap(p);
1204 
1205 #if defined(TARGET_HAS_SMC) || 1
1206 
1207 #if defined(CONFIG_USER_ONLY)
1208     if (p->flags & PAGE_WRITE) {
1209         target_ulong addr;
1210         PageDesc *p2;
1211         int prot;
1212 
1213         /* force the host page as non writable (writes will have a
1214            page fault + mprotect overhead) */
1215         page_addr &= qemu_host_page_mask;
1216         prot = 0;
1217         for(addr = page_addr; addr < page_addr + qemu_host_page_size;
1218             addr += TARGET_PAGE_SIZE) {
1219 
1220             p2 = page_find (addr >> TARGET_PAGE_BITS);
1221             if (!p2)
1222                 continue;
1223             prot |= p2->flags;
1224             p2->flags &= ~PAGE_WRITE;
1225           }
1226         mprotect(g2h(page_addr), qemu_host_page_size,
1227                  (prot & PAGE_BITS) & ~PAGE_WRITE);
1228 #ifdef DEBUG_TB_INVALIDATE
1229         printf("protecting code page: 0x" TARGET_FMT_lx "\n",
1230                page_addr);
1231 #endif
1232     }
1233 #else
1234     /* if some code is already present, then the pages are already
1235        protected. So we handle the case where only the first TB is
1236        allocated in a physical page */
1237     if (!last_first_tb) {
1238         tlb_protect_code(page_addr);
1239     }
1240 #endif
1241 
1242 #endif /* TARGET_HAS_SMC */
1243 }
1244 
1245 /* Allocate a new translation block. Flush the translation buffer if
1246    too many translation blocks or too much generated code. */
tb_alloc(target_ulong pc)1247 TranslationBlock *tb_alloc(target_ulong pc)
1248 {
1249     TranslationBlock *tb;
1250 
1251     if (nb_tbs >= code_gen_max_blocks ||
1252         (code_gen_ptr - code_gen_buffer) >= code_gen_buffer_max_size)
1253         return NULL;
1254     tb = &tbs[nb_tbs++];
1255     tb->pc = pc;
1256     tb->cflags = 0;
1257     return tb;
1258 }
1259 
tb_free(TranslationBlock * tb)1260 void tb_free(TranslationBlock *tb)
1261 {
1262     /* In practice this is mostly used for single use temporary TB
1263        Ignore the hard cases and just back up if this TB happens to
1264        be the last one generated.  */
1265     if (nb_tbs > 0 && tb == &tbs[nb_tbs - 1]) {
1266         code_gen_ptr = tb->tc_ptr;
1267         nb_tbs--;
1268     }
1269 }
1270 
1271 /* add a new TB and link it to the physical page tables. phys_page2 is
1272    (-1) to indicate that only one page contains the TB. */
tb_link_page(TranslationBlock * tb,tb_page_addr_t phys_pc,tb_page_addr_t phys_page2)1273 void tb_link_page(TranslationBlock *tb,
1274                   tb_page_addr_t phys_pc, tb_page_addr_t phys_page2)
1275 {
1276     unsigned int h;
1277     TranslationBlock **ptb;
1278 
1279     /* Grab the mmap lock to stop another thread invalidating this TB
1280        before we are done.  */
1281     mmap_lock();
1282     /* add in the physical hash table */
1283     h = tb_phys_hash_func(phys_pc);
1284     ptb = &tb_phys_hash[h];
1285     tb->phys_hash_next = *ptb;
1286     *ptb = tb;
1287 
1288     /* add in the page list */
1289     tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK);
1290     if (phys_page2 != -1)
1291         tb_alloc_page(tb, 1, phys_page2);
1292     else
1293         tb->page_addr[1] = -1;
1294 
1295     tb->jmp_first = (TranslationBlock *)((long)tb | 2);
1296     tb->jmp_next[0] = NULL;
1297     tb->jmp_next[1] = NULL;
1298 
1299     /* init original jump addresses */
1300     if (tb->tb_next_offset[0] != 0xffff)
1301         tb_reset_jump(tb, 0);
1302     if (tb->tb_next_offset[1] != 0xffff)
1303         tb_reset_jump(tb, 1);
1304 
1305 #ifdef DEBUG_TB_CHECK
1306     tb_page_check();
1307 #endif
1308     mmap_unlock();
1309 }
1310 
1311 /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1312    tb[1].tc_ptr. Return NULL if not found */
tb_find_pc(unsigned long tc_ptr)1313 TranslationBlock *tb_find_pc(unsigned long tc_ptr)
1314 {
1315     int m_min, m_max, m;
1316     unsigned long v;
1317     TranslationBlock *tb;
1318 
1319     if (nb_tbs <= 0)
1320         return NULL;
1321     if (tc_ptr < (unsigned long)code_gen_buffer ||
1322         tc_ptr >= (unsigned long)code_gen_ptr)
1323         return NULL;
1324     /* binary search (cf Knuth) */
1325     m_min = 0;
1326     m_max = nb_tbs - 1;
1327     while (m_min <= m_max) {
1328         m = (m_min + m_max) >> 1;
1329         tb = &tbs[m];
1330         v = (unsigned long)tb->tc_ptr;
1331         if (v == tc_ptr)
1332             return tb;
1333         else if (tc_ptr < v) {
1334             m_max = m - 1;
1335         } else {
1336             m_min = m + 1;
1337         }
1338     }
1339     return &tbs[m_max];
1340 }
1341 
1342 static void tb_reset_jump_recursive(TranslationBlock *tb);
1343 
tb_reset_jump_recursive2(TranslationBlock * tb,int n)1344 static inline void tb_reset_jump_recursive2(TranslationBlock *tb, int n)
1345 {
1346     TranslationBlock *tb1, *tb_next, **ptb;
1347     unsigned int n1;
1348 
1349     tb1 = tb->jmp_next[n];
1350     if (tb1 != NULL) {
1351         /* find head of list */
1352         for(;;) {
1353             n1 = (long)tb1 & 3;
1354             tb1 = (TranslationBlock *)((long)tb1 & ~3);
1355             if (n1 == 2)
1356                 break;
1357             tb1 = tb1->jmp_next[n1];
1358         }
1359         /* we are now sure now that tb jumps to tb1 */
1360         tb_next = tb1;
1361 
1362         /* remove tb from the jmp_first list */
1363         ptb = &tb_next->jmp_first;
1364         for(;;) {
1365             tb1 = *ptb;
1366             n1 = (long)tb1 & 3;
1367             tb1 = (TranslationBlock *)((long)tb1 & ~3);
1368             if (n1 == n && tb1 == tb)
1369                 break;
1370             ptb = &tb1->jmp_next[n1];
1371         }
1372         *ptb = tb->jmp_next[n];
1373         tb->jmp_next[n] = NULL;
1374 
1375         /* suppress the jump to next tb in generated code */
1376         tb_reset_jump(tb, n);
1377 
1378         /* suppress jumps in the tb on which we could have jumped */
1379         tb_reset_jump_recursive(tb_next);
1380     }
1381 }
1382 
tb_reset_jump_recursive(TranslationBlock * tb)1383 static void tb_reset_jump_recursive(TranslationBlock *tb)
1384 {
1385     tb_reset_jump_recursive2(tb, 0);
1386     tb_reset_jump_recursive2(tb, 1);
1387 }
1388 
1389 #if defined(TARGET_HAS_ICE)
1390 #if defined(CONFIG_USER_ONLY)
breakpoint_invalidate(CPUState * env,target_ulong pc)1391 static void breakpoint_invalidate(CPUState *env, target_ulong pc)
1392 {
1393     tb_invalidate_phys_page_range(pc, pc + 1, 0);
1394 }
1395 #else
breakpoint_invalidate(CPUState * env,target_ulong pc)1396 static void breakpoint_invalidate(CPUState *env, target_ulong pc)
1397 {
1398     target_phys_addr_t addr;
1399     target_ulong pd;
1400     ram_addr_t ram_addr;
1401     PhysPageDesc *p;
1402 
1403     addr = cpu_get_phys_page_debug(env, pc);
1404     p = phys_page_find(addr >> TARGET_PAGE_BITS);
1405     if (!p) {
1406         pd = IO_MEM_UNASSIGNED;
1407     } else {
1408         pd = p->phys_offset;
1409     }
1410     ram_addr = (pd & TARGET_PAGE_MASK) | (pc & ~TARGET_PAGE_MASK);
1411     tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0);
1412 }
1413 #endif
1414 #endif /* TARGET_HAS_ICE */
1415 
1416 #if defined(CONFIG_USER_ONLY)
cpu_watchpoint_remove_all(CPUState * env,int mask)1417 void cpu_watchpoint_remove_all(CPUState *env, int mask)
1418 
1419 {
1420 }
1421 
cpu_watchpoint_insert(CPUState * env,target_ulong addr,target_ulong len,int flags,CPUWatchpoint ** watchpoint)1422 int cpu_watchpoint_insert(CPUState *env, target_ulong addr, target_ulong len,
1423                           int flags, CPUWatchpoint **watchpoint)
1424 {
1425     return -ENOSYS;
1426 }
1427 #else
1428 /* Add a watchpoint.  */
cpu_watchpoint_insert(CPUState * env,target_ulong addr,target_ulong len,int flags,CPUWatchpoint ** watchpoint)1429 int cpu_watchpoint_insert(CPUState *env, target_ulong addr, target_ulong len,
1430                           int flags, CPUWatchpoint **watchpoint)
1431 {
1432     target_ulong len_mask = ~(len - 1);
1433     CPUWatchpoint *wp;
1434 
1435     /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
1436     if ((len != 1 && len != 2 && len != 4 && len != 8) || (addr & ~len_mask)) {
1437         fprintf(stderr, "qemu: tried to set invalid watchpoint at "
1438                 TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
1439         return -EINVAL;
1440     }
1441     wp = qemu_malloc(sizeof(*wp));
1442 
1443     wp->vaddr = addr;
1444     wp->len_mask = len_mask;
1445     wp->flags = flags;
1446 
1447     /* keep all GDB-injected watchpoints in front */
1448     if (flags & BP_GDB)
1449         QTAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
1450     else
1451         QTAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
1452 
1453     tlb_flush_page(env, addr);
1454 
1455     if (watchpoint)
1456         *watchpoint = wp;
1457     return 0;
1458 }
1459 
1460 /* Remove a specific watchpoint.  */
cpu_watchpoint_remove(CPUState * env,target_ulong addr,target_ulong len,int flags)1461 int cpu_watchpoint_remove(CPUState *env, target_ulong addr, target_ulong len,
1462                           int flags)
1463 {
1464     target_ulong len_mask = ~(len - 1);
1465     CPUWatchpoint *wp;
1466 
1467     QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1468         if (addr == wp->vaddr && len_mask == wp->len_mask
1469                 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
1470             cpu_watchpoint_remove_by_ref(env, wp);
1471             return 0;
1472         }
1473     }
1474     return -ENOENT;
1475 }
1476 
1477 /* Remove a specific watchpoint by reference.  */
cpu_watchpoint_remove_by_ref(CPUState * env,CPUWatchpoint * watchpoint)1478 void cpu_watchpoint_remove_by_ref(CPUState *env, CPUWatchpoint *watchpoint)
1479 {
1480     QTAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
1481 
1482     tlb_flush_page(env, watchpoint->vaddr);
1483 
1484     qemu_free(watchpoint);
1485 }
1486 
1487 /* Remove all matching watchpoints.  */
cpu_watchpoint_remove_all(CPUState * env,int mask)1488 void cpu_watchpoint_remove_all(CPUState *env, int mask)
1489 {
1490     CPUWatchpoint *wp, *next;
1491 
1492     QTAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
1493         if (wp->flags & mask)
1494             cpu_watchpoint_remove_by_ref(env, wp);
1495     }
1496 }
1497 #endif
1498 
1499 /* Add a breakpoint.  */
cpu_breakpoint_insert(CPUState * env,target_ulong pc,int flags,CPUBreakpoint ** breakpoint)1500 int cpu_breakpoint_insert(CPUState *env, target_ulong pc, int flags,
1501                           CPUBreakpoint **breakpoint)
1502 {
1503 #if defined(TARGET_HAS_ICE)
1504     CPUBreakpoint *bp;
1505 
1506     bp = qemu_malloc(sizeof(*bp));
1507 
1508     bp->pc = pc;
1509     bp->flags = flags;
1510 
1511     /* keep all GDB-injected breakpoints in front */
1512     if (flags & BP_GDB)
1513         QTAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
1514     else
1515         QTAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
1516 
1517     breakpoint_invalidate(env, pc);
1518 
1519     if (breakpoint)
1520         *breakpoint = bp;
1521     return 0;
1522 #else
1523     return -ENOSYS;
1524 #endif
1525 }
1526 
1527 /* Remove a specific breakpoint.  */
cpu_breakpoint_remove(CPUState * env,target_ulong pc,int flags)1528 int cpu_breakpoint_remove(CPUState *env, target_ulong pc, int flags)
1529 {
1530 #if defined(TARGET_HAS_ICE)
1531     CPUBreakpoint *bp;
1532 
1533     QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
1534         if (bp->pc == pc && bp->flags == flags) {
1535             cpu_breakpoint_remove_by_ref(env, bp);
1536             return 0;
1537         }
1538     }
1539     return -ENOENT;
1540 #else
1541     return -ENOSYS;
1542 #endif
1543 }
1544 
1545 /* Remove a specific breakpoint by reference.  */
cpu_breakpoint_remove_by_ref(CPUState * env,CPUBreakpoint * breakpoint)1546 void cpu_breakpoint_remove_by_ref(CPUState *env, CPUBreakpoint *breakpoint)
1547 {
1548 #if defined(TARGET_HAS_ICE)
1549     QTAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
1550 
1551     breakpoint_invalidate(env, breakpoint->pc);
1552 
1553     qemu_free(breakpoint);
1554 #endif
1555 }
1556 
1557 /* Remove all matching breakpoints. */
cpu_breakpoint_remove_all(CPUState * env,int mask)1558 void cpu_breakpoint_remove_all(CPUState *env, int mask)
1559 {
1560 #if defined(TARGET_HAS_ICE)
1561     CPUBreakpoint *bp, *next;
1562 
1563     QTAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
1564         if (bp->flags & mask)
1565             cpu_breakpoint_remove_by_ref(env, bp);
1566     }
1567 #endif
1568 }
1569 
1570 /* enable or disable single step mode. EXCP_DEBUG is returned by the
1571    CPU loop after each instruction */
cpu_single_step(CPUState * env,int enabled)1572 void cpu_single_step(CPUState *env, int enabled)
1573 {
1574 #if defined(TARGET_HAS_ICE)
1575     if (env->singlestep_enabled != enabled) {
1576         env->singlestep_enabled = enabled;
1577         if (kvm_enabled())
1578             kvm_update_guest_debug(env, 0);
1579         else {
1580             /* must flush all the translated code to avoid inconsistencies */
1581             /* XXX: only flush what is necessary */
1582             tb_flush(env);
1583         }
1584     }
1585 #endif
1586 }
1587 
1588 /* enable or disable low levels log */
cpu_set_log(int log_flags)1589 void cpu_set_log(int log_flags)
1590 {
1591     loglevel = log_flags;
1592     if (loglevel && !logfile) {
1593         logfile = fopen(logfilename, log_append ? "a" : "w");
1594         if (!logfile) {
1595             perror(logfilename);
1596             _exit(1);
1597         }
1598 #if !defined(CONFIG_SOFTMMU)
1599         /* must avoid mmap() usage of glibc by setting a buffer "by hand" */
1600         {
1601             static char logfile_buf[4096];
1602             setvbuf(logfile, logfile_buf, _IOLBF, sizeof(logfile_buf));
1603         }
1604 #elif !defined(_WIN32)
1605         /* Win32 doesn't support line-buffering and requires size >= 2 */
1606         setvbuf(logfile, NULL, _IOLBF, 0);
1607 #endif
1608         log_append = 1;
1609     }
1610     if (!loglevel && logfile) {
1611         fclose(logfile);
1612         logfile = NULL;
1613     }
1614 }
1615 
cpu_set_log_filename(const char * filename)1616 void cpu_set_log_filename(const char *filename)
1617 {
1618     logfilename = strdup(filename);
1619     if (logfile) {
1620         fclose(logfile);
1621         logfile = NULL;
1622     }
1623     cpu_set_log(loglevel);
1624 }
1625 
cpu_unlink_tb(CPUState * env)1626 static void cpu_unlink_tb(CPUState *env)
1627 {
1628     /* FIXME: TB unchaining isn't SMP safe.  For now just ignore the
1629        problem and hope the cpu will stop of its own accord.  For userspace
1630        emulation this often isn't actually as bad as it sounds.  Often
1631        signals are used primarily to interrupt blocking syscalls.  */
1632     TranslationBlock *tb;
1633     static spinlock_t interrupt_lock = SPIN_LOCK_UNLOCKED;
1634 
1635     spin_lock(&interrupt_lock);
1636     tb = env->current_tb;
1637     /* if the cpu is currently executing code, we must unlink it and
1638        all the potentially executing TB */
1639     if (tb) {
1640         env->current_tb = NULL;
1641         tb_reset_jump_recursive(tb);
1642     }
1643     spin_unlock(&interrupt_lock);
1644 }
1645 
1646 /* mask must never be zero, except for A20 change call */
cpu_interrupt(CPUState * env,int mask)1647 void cpu_interrupt(CPUState *env, int mask)
1648 {
1649     int old_mask;
1650 
1651     old_mask = env->interrupt_request;
1652     env->interrupt_request |= mask;
1653     if (kvm_enabled() && !kvm_irqchip_in_kernel())
1654 	kvm_update_interrupt_request(env);
1655 
1656 #ifndef CONFIG_USER_ONLY
1657     /*
1658      * If called from iothread context, wake the target cpu in
1659      * case its halted.
1660      */
1661     if (!qemu_cpu_self(env)) {
1662         qemu_cpu_kick(env);
1663         return;
1664     }
1665 #endif
1666 
1667     if (use_icount) {
1668         env->icount_decr.u16.high = 0xffff;
1669 #ifndef CONFIG_USER_ONLY
1670         if (!can_do_io(env)
1671             && (mask & ~old_mask) != 0) {
1672             cpu_abort(env, "Raised interrupt while not in I/O function");
1673         }
1674 #endif
1675     } else {
1676         cpu_unlink_tb(env);
1677     }
1678 }
1679 
cpu_reset_interrupt(CPUState * env,int mask)1680 void cpu_reset_interrupt(CPUState *env, int mask)
1681 {
1682     env->interrupt_request &= ~mask;
1683 }
1684 
cpu_exit(CPUState * env)1685 void cpu_exit(CPUState *env)
1686 {
1687     env->exit_request = 1;
1688     cpu_unlink_tb(env);
1689 }
1690 
1691 const CPULogItem cpu_log_items[] = {
1692     { CPU_LOG_TB_OUT_ASM, "out_asm",
1693       "show generated host assembly code for each compiled TB" },
1694     { CPU_LOG_TB_IN_ASM, "in_asm",
1695       "show target assembly code for each compiled TB" },
1696     { CPU_LOG_TB_OP, "op",
1697       "show micro ops for each compiled TB" },
1698     { CPU_LOG_TB_OP_OPT, "op_opt",
1699       "show micro ops "
1700 #ifdef TARGET_I386
1701       "before eflags optimization and "
1702 #endif
1703       "after liveness analysis" },
1704     { CPU_LOG_INT, "int",
1705       "show interrupts/exceptions in short format" },
1706     { CPU_LOG_EXEC, "exec",
1707       "show trace before each executed TB (lots of logs)" },
1708     { CPU_LOG_TB_CPU, "cpu",
1709       "show CPU state before block translation" },
1710 #ifdef TARGET_I386
1711     { CPU_LOG_PCALL, "pcall",
1712       "show protected mode far calls/returns/exceptions" },
1713     { CPU_LOG_RESET, "cpu_reset",
1714       "show CPU state before CPU resets" },
1715 #endif
1716 #ifdef DEBUG_IOPORT
1717     { CPU_LOG_IOPORT, "ioport",
1718       "show all i/o ports accesses" },
1719 #endif
1720     { 0, NULL, NULL },
1721 };
1722 
1723 #ifndef CONFIG_USER_ONLY
1724 static QLIST_HEAD(memory_client_list, CPUPhysMemoryClient) memory_client_list
1725     = QLIST_HEAD_INITIALIZER(memory_client_list);
1726 
cpu_notify_set_memory(target_phys_addr_t start_addr,ram_addr_t size,ram_addr_t phys_offset)1727 static void cpu_notify_set_memory(target_phys_addr_t start_addr,
1728                                   ram_addr_t size,
1729                                   ram_addr_t phys_offset)
1730 {
1731     CPUPhysMemoryClient *client;
1732     QLIST_FOREACH(client, &memory_client_list, list) {
1733         client->set_memory(client, start_addr, size, phys_offset);
1734     }
1735 }
1736 
cpu_notify_sync_dirty_bitmap(target_phys_addr_t start,target_phys_addr_t end)1737 static int cpu_notify_sync_dirty_bitmap(target_phys_addr_t start,
1738                                         target_phys_addr_t end)
1739 {
1740     CPUPhysMemoryClient *client;
1741     QLIST_FOREACH(client, &memory_client_list, list) {
1742         int r = client->sync_dirty_bitmap(client, start, end);
1743         if (r < 0)
1744             return r;
1745     }
1746     return 0;
1747 }
1748 
cpu_notify_migration_log(int enable)1749 static int cpu_notify_migration_log(int enable)
1750 {
1751     CPUPhysMemoryClient *client;
1752     QLIST_FOREACH(client, &memory_client_list, list) {
1753         int r = client->migration_log(client, enable);
1754         if (r < 0)
1755             return r;
1756     }
1757     return 0;
1758 }
1759 
phys_page_for_each_1(CPUPhysMemoryClient * client,int level,void ** lp)1760 static void phys_page_for_each_1(CPUPhysMemoryClient *client,
1761                                  int level, void **lp)
1762 {
1763     int i;
1764 
1765     if (*lp == NULL) {
1766         return;
1767     }
1768     if (level == 0) {
1769         PhysPageDesc *pd = *lp;
1770         for (i = 0; i < L2_SIZE; ++i) {
1771             if (pd[i].phys_offset != IO_MEM_UNASSIGNED) {
1772                 client->set_memory(client, pd[i].region_offset,
1773                                    TARGET_PAGE_SIZE, pd[i].phys_offset);
1774             }
1775         }
1776     } else {
1777         void **pp = *lp;
1778         for (i = 0; i < L2_SIZE; ++i) {
1779             phys_page_for_each_1(client, level - 1, pp + i);
1780         }
1781     }
1782 }
1783 
phys_page_for_each(CPUPhysMemoryClient * client)1784 static void phys_page_for_each(CPUPhysMemoryClient *client)
1785 {
1786     int i;
1787     for (i = 0; i < P_L1_SIZE; ++i) {
1788         phys_page_for_each_1(client, P_L1_SHIFT / L2_BITS - 1,
1789                              l1_phys_map + 1);
1790     }
1791 }
1792 
cpu_register_phys_memory_client(CPUPhysMemoryClient * client)1793 void cpu_register_phys_memory_client(CPUPhysMemoryClient *client)
1794 {
1795     QLIST_INSERT_HEAD(&memory_client_list, client, list);
1796     phys_page_for_each(client);
1797 }
1798 
cpu_unregister_phys_memory_client(CPUPhysMemoryClient * client)1799 void cpu_unregister_phys_memory_client(CPUPhysMemoryClient *client)
1800 {
1801     QLIST_REMOVE(client, list);
1802 }
1803 #endif
1804 
cmp1(const char * s1,int n,const char * s2)1805 static int cmp1(const char *s1, int n, const char *s2)
1806 {
1807     if (strlen(s2) != n)
1808         return 0;
1809     return memcmp(s1, s2, n) == 0;
1810 }
1811 
1812 /* takes a comma separated list of log masks. Return 0 if error. */
cpu_str_to_log_mask(const char * str)1813 int cpu_str_to_log_mask(const char *str)
1814 {
1815     const CPULogItem *item;
1816     int mask;
1817     const char *p, *p1;
1818 
1819     p = str;
1820     mask = 0;
1821     for(;;) {
1822         p1 = strchr(p, ',');
1823         if (!p1)
1824             p1 = p + strlen(p);
1825         if(cmp1(p,p1-p,"all")) {
1826             for(item = cpu_log_items; item->mask != 0; item++) {
1827                 mask |= item->mask;
1828             }
1829         } else {
1830             for(item = cpu_log_items; item->mask != 0; item++) {
1831                 if (cmp1(p, p1 - p, item->name))
1832                     goto found;
1833             }
1834             return 0;
1835         }
1836     found:
1837         mask |= item->mask;
1838         if (*p1 != ',')
1839             break;
1840         p = p1 + 1;
1841     }
1842     return mask;
1843 }
1844 
cpu_abort(CPUState * env,const char * fmt,...)1845 void cpu_abort(CPUState *env, const char *fmt, ...)
1846 {
1847     va_list ap;
1848     va_list ap2;
1849 
1850     va_start(ap, fmt);
1851     va_copy(ap2, ap);
1852     fprintf(stderr, "qemu: fatal: ");
1853     vfprintf(stderr, fmt, ap);
1854     fprintf(stderr, "\n");
1855 #ifdef TARGET_I386
1856     cpu_dump_state(env, stderr, fprintf, X86_DUMP_FPU | X86_DUMP_CCOP);
1857 #else
1858     cpu_dump_state(env, stderr, fprintf, 0);
1859 #endif
1860     if (qemu_log_enabled()) {
1861         qemu_log("qemu: fatal: ");
1862         qemu_log_vprintf(fmt, ap2);
1863         qemu_log("\n");
1864 #ifdef TARGET_I386
1865         log_cpu_state(env, X86_DUMP_FPU | X86_DUMP_CCOP);
1866 #else
1867         log_cpu_state(env, 0);
1868 #endif
1869         qemu_log_flush();
1870         qemu_log_close();
1871     }
1872     va_end(ap2);
1873     va_end(ap);
1874 #if defined(CONFIG_USER_ONLY)
1875     {
1876         struct sigaction act;
1877         sigfillset(&act.sa_mask);
1878         act.sa_handler = SIG_DFL;
1879         sigaction(SIGABRT, &act, NULL);
1880     }
1881 #endif
1882     abort();
1883 }
1884 
cpu_copy(CPUState * env)1885 CPUState *cpu_copy(CPUState *env)
1886 {
1887     CPUState *new_env = cpu_init(env->cpu_model_str);
1888     CPUState *next_cpu = new_env->next_cpu;
1889     int cpu_index = new_env->cpu_index;
1890 #if defined(TARGET_HAS_ICE)
1891     CPUBreakpoint *bp;
1892     CPUWatchpoint *wp;
1893 #endif
1894 
1895     memcpy(new_env, env, sizeof(CPUState));
1896 
1897     /* Preserve chaining and index. */
1898     new_env->next_cpu = next_cpu;
1899     new_env->cpu_index = cpu_index;
1900 
1901     /* Clone all break/watchpoints.
1902        Note: Once we support ptrace with hw-debug register access, make sure
1903        BP_CPU break/watchpoints are handled correctly on clone. */
1904     QTAILQ_INIT(&env->breakpoints);
1905     QTAILQ_INIT(&env->watchpoints);
1906 #if defined(TARGET_HAS_ICE)
1907     QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
1908         cpu_breakpoint_insert(new_env, bp->pc, bp->flags, NULL);
1909     }
1910     QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1911         cpu_watchpoint_insert(new_env, wp->vaddr, (~wp->len_mask) + 1,
1912                               wp->flags, NULL);
1913     }
1914 #endif
1915 
1916     return new_env;
1917 }
1918 
1919 #if !defined(CONFIG_USER_ONLY)
1920 
tlb_flush_jmp_cache(CPUState * env,target_ulong addr)1921 static inline void tlb_flush_jmp_cache(CPUState *env, target_ulong addr)
1922 {
1923     unsigned int i;
1924 
1925     /* Discard jump cache entries for any tb which might potentially
1926        overlap the flushed page.  */
1927     i = tb_jmp_cache_hash_page(addr - TARGET_PAGE_SIZE);
1928     memset (&env->tb_jmp_cache[i], 0,
1929             TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1930 
1931     i = tb_jmp_cache_hash_page(addr);
1932     memset (&env->tb_jmp_cache[i], 0,
1933             TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1934 }
1935 
1936 static CPUTLBEntry s_cputlb_empty_entry = {
1937     .addr_read  = -1,
1938     .addr_write = -1,
1939     .addr_code  = -1,
1940     .addend     = -1,
1941 };
1942 
1943 /* NOTE: if flush_global is true, also flush global entries (not
1944    implemented yet) */
tlb_flush(CPUState * env,int flush_global)1945 void tlb_flush(CPUState *env, int flush_global)
1946 {
1947     int i;
1948 
1949 #if defined(DEBUG_TLB)
1950     printf("tlb_flush:\n");
1951 #endif
1952     /* must reset current TB so that interrupts cannot modify the
1953        links while we are modifying them */
1954     env->current_tb = NULL;
1955 
1956     for(i = 0; i < CPU_TLB_SIZE; i++) {
1957         int mmu_idx;
1958         for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
1959             env->tlb_table[mmu_idx][i] = s_cputlb_empty_entry;
1960         }
1961     }
1962 
1963     memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
1964 
1965     env->tlb_flush_addr = -1;
1966     env->tlb_flush_mask = 0;
1967     tlb_flush_count++;
1968 }
1969 
tlb_flush_entry(CPUTLBEntry * tlb_entry,target_ulong addr)1970 static inline void tlb_flush_entry(CPUTLBEntry *tlb_entry, target_ulong addr)
1971 {
1972     if (addr == (tlb_entry->addr_read &
1973                  (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
1974         addr == (tlb_entry->addr_write &
1975                  (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
1976         addr == (tlb_entry->addr_code &
1977                  (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
1978         *tlb_entry = s_cputlb_empty_entry;
1979     }
1980 }
1981 
tlb_flush_page(CPUState * env,target_ulong addr)1982 void tlb_flush_page(CPUState *env, target_ulong addr)
1983 {
1984     int i;
1985     int mmu_idx;
1986 
1987 #if defined(DEBUG_TLB)
1988     printf("tlb_flush_page: " TARGET_FMT_lx "\n", addr);
1989 #endif
1990     /* Check if we need to flush due to large pages.  */
1991     if ((addr & env->tlb_flush_mask) == env->tlb_flush_addr) {
1992 #if defined(DEBUG_TLB)
1993         printf("tlb_flush_page: forced full flush ("
1994                TARGET_FMT_lx "/" TARGET_FMT_lx ")\n",
1995                env->tlb_flush_addr, env->tlb_flush_mask);
1996 #endif
1997         tlb_flush(env, 1);
1998         return;
1999     }
2000     /* must reset current TB so that interrupts cannot modify the
2001        links while we are modifying them */
2002     env->current_tb = NULL;
2003 
2004     addr &= TARGET_PAGE_MASK;
2005     i = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
2006     for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++)
2007         tlb_flush_entry(&env->tlb_table[mmu_idx][i], addr);
2008 
2009     tlb_flush_jmp_cache(env, addr);
2010 }
2011 
2012 /* update the TLBs so that writes to code in the virtual page 'addr'
2013    can be detected */
tlb_protect_code(ram_addr_t ram_addr)2014 static void tlb_protect_code(ram_addr_t ram_addr)
2015 {
2016     cpu_physical_memory_reset_dirty(ram_addr,
2017                                     ram_addr + TARGET_PAGE_SIZE,
2018                                     CODE_DIRTY_FLAG);
2019 }
2020 
2021 /* update the TLB so that writes in physical page 'phys_addr' are no longer
2022    tested for self modifying code */
tlb_unprotect_code_phys(CPUState * env,ram_addr_t ram_addr,target_ulong vaddr)2023 static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr,
2024                                     target_ulong vaddr)
2025 {
2026     cpu_physical_memory_set_dirty_flags(ram_addr, CODE_DIRTY_FLAG);
2027 }
2028 
tlb_reset_dirty_range(CPUTLBEntry * tlb_entry,unsigned long start,unsigned long length)2029 static inline void tlb_reset_dirty_range(CPUTLBEntry *tlb_entry,
2030                                          unsigned long start, unsigned long length)
2031 {
2032     unsigned long addr;
2033     if ((tlb_entry->addr_write & ~TARGET_PAGE_MASK) == IO_MEM_RAM) {
2034         addr = (tlb_entry->addr_write & TARGET_PAGE_MASK) + tlb_entry->addend;
2035         if ((addr - start) < length) {
2036             tlb_entry->addr_write = (tlb_entry->addr_write & TARGET_PAGE_MASK) | TLB_NOTDIRTY;
2037         }
2038     }
2039 }
2040 
2041 /* Note: start and end must be within the same ram block.  */
cpu_physical_memory_reset_dirty(ram_addr_t start,ram_addr_t end,int dirty_flags)2042 void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
2043                                      int dirty_flags)
2044 {
2045     CPUState *env;
2046     unsigned long length, start1;
2047     int i;
2048 
2049     start &= TARGET_PAGE_MASK;
2050     end = TARGET_PAGE_ALIGN(end);
2051 
2052     length = end - start;
2053     if (length == 0)
2054         return;
2055     cpu_physical_memory_mask_dirty_range(start, length, dirty_flags);
2056 
2057     /* we modify the TLB cache so that the dirty bit will be set again
2058        when accessing the range */
2059     start1 = (unsigned long)qemu_safe_ram_ptr(start);
2060     /* Chek that we don't span multiple blocks - this breaks the
2061        address comparisons below.  */
2062     if ((unsigned long)qemu_safe_ram_ptr(end - 1) - start1
2063             != (end - 1) - start) {
2064         abort();
2065     }
2066 
2067     for(env = first_cpu; env != NULL; env = env->next_cpu) {
2068         int mmu_idx;
2069         for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
2070             for(i = 0; i < CPU_TLB_SIZE; i++)
2071                 tlb_reset_dirty_range(&env->tlb_table[mmu_idx][i],
2072                                       start1, length);
2073         }
2074     }
2075 }
2076 
cpu_physical_memory_set_dirty_tracking(int enable)2077 int cpu_physical_memory_set_dirty_tracking(int enable)
2078 {
2079     int ret = 0;
2080     in_migration = enable;
2081     ret = cpu_notify_migration_log(!!enable);
2082     return ret;
2083 }
2084 
cpu_physical_memory_get_dirty_tracking(void)2085 int cpu_physical_memory_get_dirty_tracking(void)
2086 {
2087     return in_migration;
2088 }
2089 
cpu_physical_sync_dirty_bitmap(target_phys_addr_t start_addr,target_phys_addr_t end_addr)2090 int cpu_physical_sync_dirty_bitmap(target_phys_addr_t start_addr,
2091                                    target_phys_addr_t end_addr)
2092 {
2093     int ret;
2094 
2095     ret = cpu_notify_sync_dirty_bitmap(start_addr, end_addr);
2096     return ret;
2097 }
2098 
tlb_update_dirty(CPUTLBEntry * tlb_entry)2099 static inline void tlb_update_dirty(CPUTLBEntry *tlb_entry)
2100 {
2101     ram_addr_t ram_addr;
2102     void *p;
2103 
2104     if ((tlb_entry->addr_write & ~TARGET_PAGE_MASK) == IO_MEM_RAM) {
2105         p = (void *)(unsigned long)((tlb_entry->addr_write & TARGET_PAGE_MASK)
2106             + tlb_entry->addend);
2107         ram_addr = qemu_ram_addr_from_host_nofail(p);
2108         if (!cpu_physical_memory_is_dirty(ram_addr)) {
2109             tlb_entry->addr_write |= TLB_NOTDIRTY;
2110         }
2111     }
2112 }
2113 
2114 /* update the TLB according to the current state of the dirty bits */
cpu_tlb_update_dirty(CPUState * env)2115 void cpu_tlb_update_dirty(CPUState *env)
2116 {
2117     int i;
2118     int mmu_idx;
2119     for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
2120         for(i = 0; i < CPU_TLB_SIZE; i++)
2121             tlb_update_dirty(&env->tlb_table[mmu_idx][i]);
2122     }
2123 }
2124 
tlb_set_dirty1(CPUTLBEntry * tlb_entry,target_ulong vaddr)2125 static inline void tlb_set_dirty1(CPUTLBEntry *tlb_entry, target_ulong vaddr)
2126 {
2127     if (tlb_entry->addr_write == (vaddr | TLB_NOTDIRTY))
2128         tlb_entry->addr_write = vaddr;
2129 }
2130 
2131 /* update the TLB corresponding to virtual page vaddr
2132    so that it is no longer dirty */
tlb_set_dirty(CPUState * env,target_ulong vaddr)2133 static inline void tlb_set_dirty(CPUState *env, target_ulong vaddr)
2134 {
2135     int i;
2136     int mmu_idx;
2137 
2138     vaddr &= TARGET_PAGE_MASK;
2139     i = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
2140     for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++)
2141         tlb_set_dirty1(&env->tlb_table[mmu_idx][i], vaddr);
2142 }
2143 
2144 /* Our TLB does not support large pages, so remember the area covered by
2145    large pages and trigger a full TLB flush if these are invalidated.  */
tlb_add_large_page(CPUState * env,target_ulong vaddr,target_ulong size)2146 static void tlb_add_large_page(CPUState *env, target_ulong vaddr,
2147                                target_ulong size)
2148 {
2149     target_ulong mask = ~(size - 1);
2150 
2151     if (env->tlb_flush_addr == (target_ulong)-1) {
2152         env->tlb_flush_addr = vaddr & mask;
2153         env->tlb_flush_mask = mask;
2154         return;
2155     }
2156     /* Extend the existing region to include the new page.
2157        This is a compromise between unnecessary flushes and the cost
2158        of maintaining a full variable size TLB.  */
2159     mask &= env->tlb_flush_mask;
2160     while (((env->tlb_flush_addr ^ vaddr) & mask) != 0) {
2161         mask <<= 1;
2162     }
2163     env->tlb_flush_addr &= mask;
2164     env->tlb_flush_mask = mask;
2165 }
2166 
2167 /* Add a new TLB entry. At most one entry for a given virtual address
2168    is permitted. Only a single TARGET_PAGE_SIZE region is mapped, the
2169    supplied size is only used by tlb_flush_page.  */
tlb_set_page(CPUState * env,target_ulong vaddr,target_phys_addr_t paddr,int prot,int mmu_idx,target_ulong size)2170 void tlb_set_page(CPUState *env, target_ulong vaddr,
2171                   target_phys_addr_t paddr, int prot,
2172                   int mmu_idx, target_ulong size)
2173 {
2174     PhysPageDesc *p;
2175     unsigned long pd;
2176     unsigned int index;
2177     target_ulong address;
2178     target_ulong code_address;
2179     unsigned long addend;
2180     CPUTLBEntry *te;
2181     CPUWatchpoint *wp;
2182     target_phys_addr_t iotlb;
2183 
2184     assert(size >= TARGET_PAGE_SIZE);
2185     if (size != TARGET_PAGE_SIZE) {
2186         tlb_add_large_page(env, vaddr, size);
2187     }
2188     p = phys_page_find(paddr >> TARGET_PAGE_BITS);
2189     if (!p) {
2190         pd = IO_MEM_UNASSIGNED;
2191     } else {
2192         pd = p->phys_offset;
2193     }
2194 #if defined(DEBUG_TLB)
2195     printf("tlb_set_page: vaddr=" TARGET_FMT_lx " paddr=0x" TARGET_FMT_plx
2196            " prot=%x idx=%d pd=0x%08lx\n",
2197            vaddr, paddr, prot, mmu_idx, pd);
2198 #endif
2199 
2200     address = vaddr;
2201     if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM && !(pd & IO_MEM_ROMD)) {
2202         /* IO memory case (romd handled later) */
2203         address |= TLB_MMIO;
2204     }
2205     addend = (unsigned long)qemu_get_ram_ptr(pd & TARGET_PAGE_MASK);
2206     if ((pd & ~TARGET_PAGE_MASK) <= IO_MEM_ROM) {
2207         /* Normal RAM.  */
2208         iotlb = pd & TARGET_PAGE_MASK;
2209         if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM)
2210             iotlb |= IO_MEM_NOTDIRTY;
2211         else
2212             iotlb |= IO_MEM_ROM;
2213     } else {
2214         /* IO handlers are currently passed a physical address.
2215            It would be nice to pass an offset from the base address
2216            of that region.  This would avoid having to special case RAM,
2217            and avoid full address decoding in every device.
2218            We can't use the high bits of pd for this because
2219            IO_MEM_ROMD uses these as a ram address.  */
2220         iotlb = (pd & ~TARGET_PAGE_MASK);
2221         if (p) {
2222             iotlb += p->region_offset;
2223         } else {
2224             iotlb += paddr;
2225         }
2226     }
2227 
2228     code_address = address;
2229     /* Make accesses to pages with watchpoints go via the
2230        watchpoint trap routines.  */
2231     QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
2232         if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
2233             /* Avoid trapping reads of pages with a write breakpoint. */
2234             if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
2235                 iotlb = io_mem_watch + paddr;
2236                 address |= TLB_MMIO;
2237                 break;
2238             }
2239         }
2240     }
2241 
2242     index = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
2243     env->iotlb[mmu_idx][index] = iotlb - vaddr;
2244     te = &env->tlb_table[mmu_idx][index];
2245     te->addend = addend - vaddr;
2246     if (prot & PAGE_READ) {
2247         te->addr_read = address;
2248     } else {
2249         te->addr_read = -1;
2250     }
2251 
2252     if (prot & PAGE_EXEC) {
2253         te->addr_code = code_address;
2254     } else {
2255         te->addr_code = -1;
2256     }
2257     if (prot & PAGE_WRITE) {
2258         if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_ROM ||
2259             (pd & IO_MEM_ROMD)) {
2260             /* Write access calls the I/O callback.  */
2261             te->addr_write = address | TLB_MMIO;
2262         } else if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM &&
2263                    !cpu_physical_memory_is_dirty(pd)) {
2264             te->addr_write = address | TLB_NOTDIRTY;
2265         } else {
2266             te->addr_write = address;
2267         }
2268     } else {
2269         te->addr_write = -1;
2270     }
2271 }
2272 
2273 #else
2274 
tlb_flush(CPUState * env,int flush_global)2275 void tlb_flush(CPUState *env, int flush_global)
2276 {
2277 }
2278 
tlb_flush_page(CPUState * env,target_ulong addr)2279 void tlb_flush_page(CPUState *env, target_ulong addr)
2280 {
2281 }
2282 
2283 /*
2284  * Walks guest process memory "regions" one by one
2285  * and calls callback function 'fn' for each region.
2286  */
2287 
2288 struct walk_memory_regions_data
2289 {
2290     walk_memory_regions_fn fn;
2291     void *priv;
2292     unsigned long start;
2293     int prot;
2294 };
2295 
walk_memory_regions_end(struct walk_memory_regions_data * data,abi_ulong end,int new_prot)2296 static int walk_memory_regions_end(struct walk_memory_regions_data *data,
2297                                    abi_ulong end, int new_prot)
2298 {
2299     if (data->start != -1ul) {
2300         int rc = data->fn(data->priv, data->start, end, data->prot);
2301         if (rc != 0) {
2302             return rc;
2303         }
2304     }
2305 
2306     data->start = (new_prot ? end : -1ul);
2307     data->prot = new_prot;
2308 
2309     return 0;
2310 }
2311 
walk_memory_regions_1(struct walk_memory_regions_data * data,abi_ulong base,int level,void ** lp)2312 static int walk_memory_regions_1(struct walk_memory_regions_data *data,
2313                                  abi_ulong base, int level, void **lp)
2314 {
2315     abi_ulong pa;
2316     int i, rc;
2317 
2318     if (*lp == NULL) {
2319         return walk_memory_regions_end(data, base, 0);
2320     }
2321 
2322     if (level == 0) {
2323         PageDesc *pd = *lp;
2324         for (i = 0; i < L2_SIZE; ++i) {
2325             int prot = pd[i].flags;
2326 
2327             pa = base | (i << TARGET_PAGE_BITS);
2328             if (prot != data->prot) {
2329                 rc = walk_memory_regions_end(data, pa, prot);
2330                 if (rc != 0) {
2331                     return rc;
2332                 }
2333             }
2334         }
2335     } else {
2336         void **pp = *lp;
2337         for (i = 0; i < L2_SIZE; ++i) {
2338             pa = base | ((abi_ulong)i <<
2339                 (TARGET_PAGE_BITS + L2_BITS * level));
2340             rc = walk_memory_regions_1(data, pa, level - 1, pp + i);
2341             if (rc != 0) {
2342                 return rc;
2343             }
2344         }
2345     }
2346 
2347     return 0;
2348 }
2349 
walk_memory_regions(void * priv,walk_memory_regions_fn fn)2350 int walk_memory_regions(void *priv, walk_memory_regions_fn fn)
2351 {
2352     struct walk_memory_regions_data data;
2353     unsigned long i;
2354 
2355     data.fn = fn;
2356     data.priv = priv;
2357     data.start = -1ul;
2358     data.prot = 0;
2359 
2360     for (i = 0; i < V_L1_SIZE; i++) {
2361         int rc = walk_memory_regions_1(&data, (abi_ulong)i << V_L1_SHIFT,
2362                                        V_L1_SHIFT / L2_BITS - 1, l1_map + i);
2363         if (rc != 0) {
2364             return rc;
2365         }
2366     }
2367 
2368     return walk_memory_regions_end(&data, 0, 0);
2369 }
2370 
dump_region(void * priv,abi_ulong start,abi_ulong end,unsigned long prot)2371 static int dump_region(void *priv, abi_ulong start,
2372     abi_ulong end, unsigned long prot)
2373 {
2374     FILE *f = (FILE *)priv;
2375 
2376     (void) fprintf(f, TARGET_ABI_FMT_lx"-"TARGET_ABI_FMT_lx
2377         " "TARGET_ABI_FMT_lx" %c%c%c\n",
2378         start, end, end - start,
2379         ((prot & PAGE_READ) ? 'r' : '-'),
2380         ((prot & PAGE_WRITE) ? 'w' : '-'),
2381         ((prot & PAGE_EXEC) ? 'x' : '-'));
2382 
2383     return (0);
2384 }
2385 
2386 /* dump memory mappings */
page_dump(FILE * f)2387 void page_dump(FILE *f)
2388 {
2389     (void) fprintf(f, "%-8s %-8s %-8s %s\n",
2390             "start", "end", "size", "prot");
2391     walk_memory_regions(f, dump_region);
2392 }
2393 
page_get_flags(target_ulong address)2394 int page_get_flags(target_ulong address)
2395 {
2396     PageDesc *p;
2397 
2398     p = page_find(address >> TARGET_PAGE_BITS);
2399     if (!p)
2400         return 0;
2401     return p->flags;
2402 }
2403 
2404 /* Modify the flags of a page and invalidate the code if necessary.
2405    The flag PAGE_WRITE_ORG is positioned automatically depending
2406    on PAGE_WRITE.  The mmap_lock should already be held.  */
page_set_flags(target_ulong start,target_ulong end,int flags)2407 void page_set_flags(target_ulong start, target_ulong end, int flags)
2408 {
2409     target_ulong addr, len;
2410 
2411     /* This function should never be called with addresses outside the
2412        guest address space.  If this assert fires, it probably indicates
2413        a missing call to h2g_valid.  */
2414 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
2415     assert(end < ((abi_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
2416 #endif
2417     assert(start < end);
2418 
2419     start = start & TARGET_PAGE_MASK;
2420     end = TARGET_PAGE_ALIGN(end);
2421 
2422     if (flags & PAGE_WRITE) {
2423         flags |= PAGE_WRITE_ORG;
2424     }
2425 
2426     for (addr = start, len = end - start;
2427          len != 0;
2428          len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
2429         PageDesc *p = page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
2430 
2431         /* If the write protection bit is set, then we invalidate
2432            the code inside.  */
2433         if (!(p->flags & PAGE_WRITE) &&
2434             (flags & PAGE_WRITE) &&
2435             p->first_tb) {
2436             tb_invalidate_phys_page(addr, 0, NULL);
2437         }
2438         p->flags = flags;
2439     }
2440 }
2441 
page_check_range(target_ulong start,target_ulong len,int flags)2442 int page_check_range(target_ulong start, target_ulong len, int flags)
2443 {
2444     PageDesc *p;
2445     target_ulong end;
2446     target_ulong addr;
2447 
2448     /* This function should never be called with addresses outside the
2449        guest address space.  If this assert fires, it probably indicates
2450        a missing call to h2g_valid.  */
2451 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
2452     assert(start < ((abi_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
2453 #endif
2454 
2455     if (len == 0) {
2456         return 0;
2457     }
2458     if (start + len - 1 < start) {
2459         /* We've wrapped around.  */
2460         return -1;
2461     }
2462 
2463     end = TARGET_PAGE_ALIGN(start+len); /* must do before we loose bits in the next step */
2464     start = start & TARGET_PAGE_MASK;
2465 
2466     for (addr = start, len = end - start;
2467          len != 0;
2468          len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
2469         p = page_find(addr >> TARGET_PAGE_BITS);
2470         if( !p )
2471             return -1;
2472         if( !(p->flags & PAGE_VALID) )
2473             return -1;
2474 
2475         if ((flags & PAGE_READ) && !(p->flags & PAGE_READ))
2476             return -1;
2477         if (flags & PAGE_WRITE) {
2478             if (!(p->flags & PAGE_WRITE_ORG))
2479                 return -1;
2480             /* unprotect the page if it was put read-only because it
2481                contains translated code */
2482             if (!(p->flags & PAGE_WRITE)) {
2483                 if (!page_unprotect(addr, 0, NULL))
2484                     return -1;
2485             }
2486             return 0;
2487         }
2488     }
2489     return 0;
2490 }
2491 
2492 /* called from signal handler: invalidate the code and unprotect the
2493    page. Return TRUE if the fault was successfully handled. */
page_unprotect(target_ulong address,unsigned long pc,void * puc)2494 int page_unprotect(target_ulong address, unsigned long pc, void *puc)
2495 {
2496     unsigned int prot;
2497     PageDesc *p;
2498     target_ulong host_start, host_end, addr;
2499 
2500     /* Technically this isn't safe inside a signal handler.  However we
2501        know this only ever happens in a synchronous SEGV handler, so in
2502        practice it seems to be ok.  */
2503     mmap_lock();
2504 
2505     p = page_find(address >> TARGET_PAGE_BITS);
2506     if (!p) {
2507         mmap_unlock();
2508         return 0;
2509     }
2510 
2511     /* if the page was really writable, then we change its
2512        protection back to writable */
2513     if ((p->flags & PAGE_WRITE_ORG) && !(p->flags & PAGE_WRITE)) {
2514         host_start = address & qemu_host_page_mask;
2515         host_end = host_start + qemu_host_page_size;
2516 
2517         prot = 0;
2518         for (addr = host_start ; addr < host_end ; addr += TARGET_PAGE_SIZE) {
2519             p = page_find(addr >> TARGET_PAGE_BITS);
2520             p->flags |= PAGE_WRITE;
2521             prot |= p->flags;
2522 
2523             /* and since the content will be modified, we must invalidate
2524                the corresponding translated code. */
2525             tb_invalidate_phys_page(addr, pc, puc);
2526 #ifdef DEBUG_TB_CHECK
2527             tb_invalidate_check(addr);
2528 #endif
2529         }
2530         mprotect((void *)g2h(host_start), qemu_host_page_size,
2531                  prot & PAGE_BITS);
2532 
2533         mmap_unlock();
2534         return 1;
2535     }
2536     mmap_unlock();
2537     return 0;
2538 }
2539 
tlb_set_dirty(CPUState * env,unsigned long addr,target_ulong vaddr)2540 static inline void tlb_set_dirty(CPUState *env,
2541                                  unsigned long addr, target_ulong vaddr)
2542 {
2543 }
2544 #endif /* defined(CONFIG_USER_ONLY) */
2545 
2546 #if !defined(CONFIG_USER_ONLY)
2547 
2548 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
2549 typedef struct subpage_t {
2550     target_phys_addr_t base;
2551     ram_addr_t sub_io_index[TARGET_PAGE_SIZE];
2552     ram_addr_t region_offset[TARGET_PAGE_SIZE];
2553 } subpage_t;
2554 
2555 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
2556                              ram_addr_t memory, ram_addr_t region_offset);
2557 static subpage_t *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
2558                                 ram_addr_t orig_memory,
2559                                 ram_addr_t region_offset);
2560 #define CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2, \
2561                       need_subpage)                                     \
2562     do {                                                                \
2563         if (addr > start_addr)                                          \
2564             start_addr2 = 0;                                            \
2565         else {                                                          \
2566             start_addr2 = start_addr & ~TARGET_PAGE_MASK;               \
2567             if (start_addr2 > 0)                                        \
2568                 need_subpage = 1;                                       \
2569         }                                                               \
2570                                                                         \
2571         if ((start_addr + orig_size) - addr >= TARGET_PAGE_SIZE)        \
2572             end_addr2 = TARGET_PAGE_SIZE - 1;                           \
2573         else {                                                          \
2574             end_addr2 = (start_addr + orig_size - 1) & ~TARGET_PAGE_MASK; \
2575             if (end_addr2 < TARGET_PAGE_SIZE - 1)                       \
2576                 need_subpage = 1;                                       \
2577         }                                                               \
2578     } while (0)
2579 
2580 /* register physical memory.
2581    For RAM, 'size' must be a multiple of the target page size.
2582    If (phys_offset & ~TARGET_PAGE_MASK) != 0, then it is an
2583    io memory page.  The address used when calling the IO function is
2584    the offset from the start of the region, plus region_offset.  Both
2585    start_addr and region_offset are rounded down to a page boundary
2586    before calculating this offset.  This should not be a problem unless
2587    the low bits of start_addr and region_offset differ.  */
cpu_register_physical_memory_offset(target_phys_addr_t start_addr,ram_addr_t size,ram_addr_t phys_offset,ram_addr_t region_offset)2588 void cpu_register_physical_memory_offset(target_phys_addr_t start_addr,
2589                                          ram_addr_t size,
2590                                          ram_addr_t phys_offset,
2591                                          ram_addr_t region_offset)
2592 {
2593     target_phys_addr_t addr, end_addr;
2594     PhysPageDesc *p;
2595     CPUState *env;
2596     ram_addr_t orig_size = size;
2597     subpage_t *subpage;
2598 
2599     cpu_notify_set_memory(start_addr, size, phys_offset);
2600 
2601     if (phys_offset == IO_MEM_UNASSIGNED) {
2602         region_offset = start_addr;
2603     }
2604     region_offset &= TARGET_PAGE_MASK;
2605     size = (size + TARGET_PAGE_SIZE - 1) & TARGET_PAGE_MASK;
2606     end_addr = start_addr + (target_phys_addr_t)size;
2607     for(addr = start_addr; addr != end_addr; addr += TARGET_PAGE_SIZE) {
2608         p = phys_page_find(addr >> TARGET_PAGE_BITS);
2609         if (p && p->phys_offset != IO_MEM_UNASSIGNED) {
2610             ram_addr_t orig_memory = p->phys_offset;
2611             target_phys_addr_t start_addr2, end_addr2;
2612             int need_subpage = 0;
2613 
2614             CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2,
2615                           need_subpage);
2616             if (need_subpage) {
2617                 if (!(orig_memory & IO_MEM_SUBPAGE)) {
2618                     subpage = subpage_init((addr & TARGET_PAGE_MASK),
2619                                            &p->phys_offset, orig_memory,
2620                                            p->region_offset);
2621                 } else {
2622                     subpage = io_mem_opaque[(orig_memory & ~TARGET_PAGE_MASK)
2623                                             >> IO_MEM_SHIFT];
2624                 }
2625                 subpage_register(subpage, start_addr2, end_addr2, phys_offset,
2626                                  region_offset);
2627                 p->region_offset = 0;
2628             } else {
2629                 p->phys_offset = phys_offset;
2630                 if ((phys_offset & ~TARGET_PAGE_MASK) <= IO_MEM_ROM ||
2631                     (phys_offset & IO_MEM_ROMD))
2632                     phys_offset += TARGET_PAGE_SIZE;
2633             }
2634         } else {
2635             p = phys_page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
2636             p->phys_offset = phys_offset;
2637             p->region_offset = region_offset;
2638             if ((phys_offset & ~TARGET_PAGE_MASK) <= IO_MEM_ROM ||
2639                 (phys_offset & IO_MEM_ROMD)) {
2640                 phys_offset += TARGET_PAGE_SIZE;
2641             } else {
2642                 target_phys_addr_t start_addr2, end_addr2;
2643                 int need_subpage = 0;
2644 
2645                 CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr,
2646                               end_addr2, need_subpage);
2647 
2648                 if (need_subpage) {
2649                     subpage = subpage_init((addr & TARGET_PAGE_MASK),
2650                                            &p->phys_offset, IO_MEM_UNASSIGNED,
2651                                            addr & TARGET_PAGE_MASK);
2652                     subpage_register(subpage, start_addr2, end_addr2,
2653                                      phys_offset, region_offset);
2654                     p->region_offset = 0;
2655                 }
2656             }
2657         }
2658         region_offset += TARGET_PAGE_SIZE;
2659     }
2660 
2661     /* since each CPU stores ram addresses in its TLB cache, we must
2662        reset the modified entries */
2663     /* XXX: slow ! */
2664     for(env = first_cpu; env != NULL; env = env->next_cpu) {
2665         tlb_flush(env, 1);
2666     }
2667 }
2668 
2669 /* XXX: temporary until new memory mapping API */
cpu_get_physical_page_desc(target_phys_addr_t addr)2670 ram_addr_t cpu_get_physical_page_desc(target_phys_addr_t addr)
2671 {
2672     PhysPageDesc *p;
2673 
2674     p = phys_page_find(addr >> TARGET_PAGE_BITS);
2675     if (!p)
2676         return IO_MEM_UNASSIGNED;
2677     return p->phys_offset;
2678 }
2679 
qemu_register_coalesced_mmio(target_phys_addr_t addr,ram_addr_t size)2680 void qemu_register_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size)
2681 {
2682     if (kvm_enabled())
2683         kvm_coalesce_mmio_region(addr, size);
2684 }
2685 
qemu_unregister_coalesced_mmio(target_phys_addr_t addr,ram_addr_t size)2686 void qemu_unregister_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size)
2687 {
2688     if (kvm_enabled())
2689         kvm_uncoalesce_mmio_region(addr, size);
2690 }
2691 
qemu_flush_coalesced_mmio_buffer(void)2692 void qemu_flush_coalesced_mmio_buffer(void)
2693 {
2694     if (kvm_enabled())
2695         kvm_flush_coalesced_mmio_buffer();
2696 }
2697 
2698 #if defined(__linux__) && !defined(TARGET_S390X)
2699 
2700 #include <sys/vfs.h>
2701 
2702 #define HUGETLBFS_MAGIC       0x958458f6
2703 
gethugepagesize(const char * path)2704 static long gethugepagesize(const char *path)
2705 {
2706     struct statfs fs;
2707     int ret;
2708 
2709     do {
2710         ret = statfs(path, &fs);
2711     } while (ret != 0 && errno == EINTR);
2712 
2713     if (ret != 0) {
2714         perror(path);
2715         return 0;
2716     }
2717 
2718     if (fs.f_type != HUGETLBFS_MAGIC)
2719         fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path);
2720 
2721     return fs.f_bsize;
2722 }
2723 
file_ram_alloc(RAMBlock * block,ram_addr_t memory,const char * path)2724 static void *file_ram_alloc(RAMBlock *block,
2725                             ram_addr_t memory,
2726                             const char *path)
2727 {
2728     char *filename;
2729     void *area;
2730     int fd;
2731 #ifdef MAP_POPULATE
2732     int flags;
2733 #endif
2734     unsigned long hpagesize;
2735 
2736     hpagesize = gethugepagesize(path);
2737     if (!hpagesize) {
2738         return NULL;
2739     }
2740 
2741     if (memory < hpagesize) {
2742         return NULL;
2743     }
2744 
2745     if (kvm_enabled() && !kvm_has_sync_mmu()) {
2746         fprintf(stderr, "host lacks kvm mmu notifiers, -mem-path unsupported\n");
2747         return NULL;
2748     }
2749 
2750     if (asprintf(&filename, "%s/qemu_back_mem.XXXXXX", path) == -1) {
2751         return NULL;
2752     }
2753 
2754     fd = mkstemp(filename);
2755     if (fd < 0) {
2756         perror("unable to create backing store for hugepages");
2757         free(filename);
2758         return NULL;
2759     }
2760     unlink(filename);
2761     free(filename);
2762 
2763     memory = (memory+hpagesize-1) & ~(hpagesize-1);
2764 
2765     /*
2766      * ftruncate is not supported by hugetlbfs in older
2767      * hosts, so don't bother bailing out on errors.
2768      * If anything goes wrong with it under other filesystems,
2769      * mmap will fail.
2770      */
2771     if (ftruncate(fd, memory))
2772         perror("ftruncate");
2773 
2774 #ifdef MAP_POPULATE
2775     /* NB: MAP_POPULATE won't exhaustively alloc all phys pages in the case
2776      * MAP_PRIVATE is requested.  For mem_prealloc we mmap as MAP_SHARED
2777      * to sidestep this quirk.
2778      */
2779     flags = mem_prealloc ? MAP_POPULATE | MAP_SHARED : MAP_PRIVATE;
2780     area = mmap(0, memory, PROT_READ | PROT_WRITE, flags, fd, 0);
2781 #else
2782     area = mmap(0, memory, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
2783 #endif
2784     if (area == MAP_FAILED) {
2785         perror("file_ram_alloc: can't mmap RAM pages");
2786         close(fd);
2787         return (NULL);
2788     }
2789     block->fd = fd;
2790     return area;
2791 }
2792 #endif
2793 
find_ram_offset(ram_addr_t size)2794 static ram_addr_t find_ram_offset(ram_addr_t size)
2795 {
2796     RAMBlock *block, *next_block;
2797     ram_addr_t offset = 0, mingap = ULONG_MAX;
2798 
2799     if (QLIST_EMPTY(&ram_list.blocks))
2800         return 0;
2801 
2802     QLIST_FOREACH(block, &ram_list.blocks, next) {
2803         ram_addr_t end, next = ULONG_MAX;
2804 
2805         end = block->offset + block->length;
2806 
2807         QLIST_FOREACH(next_block, &ram_list.blocks, next) {
2808             if (next_block->offset >= end) {
2809                 next = MIN(next, next_block->offset);
2810             }
2811         }
2812         if (next - end >= size && next - end < mingap) {
2813             offset =  end;
2814             mingap = next - end;
2815         }
2816     }
2817     return offset;
2818 }
2819 
last_ram_offset(void)2820 static ram_addr_t last_ram_offset(void)
2821 {
2822     RAMBlock *block;
2823     ram_addr_t last = 0;
2824 
2825     QLIST_FOREACH(block, &ram_list.blocks, next)
2826         last = MAX(last, block->offset + block->length);
2827 
2828     return last;
2829 }
2830 
2831 #ifdef CONFIG_SOLARIS
2832 static int
qemu_mlock(caddr_t base,ram_addr_t size)2833 qemu_mlock(caddr_t base, ram_addr_t size)
2834 {
2835 	ram_addr_t ps = qemu_real_host_page_size, nbytes, locked = 0;
2836 	ram_addr_t remaining = size / ps;
2837 	ram_addr_t step = remaining;
2838 	timespec_t tv;
2839 	hrtime_t waiting = 0, threshold;
2840 
2841 	tv.tv_sec = 0;
2842 	tv.tv_nsec = NANOSEC / MILLISEC;
2843 	threshold = 10 * (hrtime_t)NANOSEC;
2844 
2845 	/*
2846 	 * We cannot lock memory with a single call to mlock() because it
2847 	 * won't result in sustained memory pressure:  if there is a
2848 	 * substantial amount of kernel memory in use electively (e.g., for
2849 	 * the ARC) a single call to mlock() may fail where sustained memory
2850 	 * pressure would succeed.  We therefore start by trying to lock the
2851 	 * entire region, adjusting our size down as we fail with EAGAIN; once
2852 	 * we successfully lock a portion of the region, we advance to the
2853 	 * unlocked portion of the region (if any remains) and increase the
2854 	 * size.  Note that this will continue to hoard memory until it locks
2855 	 * what it needs -- it won't give up.  To help debug situations in
2856 	 * which one has mistakenly overprovisioned, we emit a message every
2857 	 * ten seconds with no forward progress.
2858 	 */
2859 	while (remaining) {
2860 		if (step > remaining)
2861 			step = remaining;
2862 
2863 		while (mlock(base, (nbytes = step * ps)) == -1) {
2864 			if (errno != EAGAIN)
2865 				return (-1);
2866 
2867 			if (waiting == 0)
2868 				waiting = gethrtime();
2869 
2870 			if (step > 1) {
2871 				step >>= 1;
2872 				continue;
2873 			}
2874 
2875 			(void) nanosleep(&tv, NULL);
2876 
2877 			if (gethrtime() - waiting > threshold) {
2878 				(void) fprintf(stderr, "qemu_mlock: have only "
2879 				    "locked %ld of %ld bytes; still "
2880 				    "trying...\n", locked, size);
2881 				waiting = 0;
2882 			}
2883 		}
2884 
2885 		waiting = 0;
2886 		base += nbytes;
2887 		locked += nbytes;
2888 		remaining -= step;
2889 
2890 		step <<= 1;
2891 	}
2892 
2893 	return (0);
2894 }
2895 #endif
2896 
qemu_ram_alloc_from_ptr(DeviceState * dev,const char * name,ram_addr_t size,void * host)2897 ram_addr_t qemu_ram_alloc_from_ptr(DeviceState *dev, const char *name,
2898                                    ram_addr_t size, void *host)
2899 {
2900     RAMBlock *new_block, *block;
2901 
2902     size = TARGET_PAGE_ALIGN(size);
2903     new_block = qemu_mallocz(sizeof(*new_block));
2904 
2905     if (dev && dev->parent_bus && dev->parent_bus->info->get_dev_path) {
2906         char *id = dev->parent_bus->info->get_dev_path(dev);
2907         if (id) {
2908             snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
2909             qemu_free(id);
2910         }
2911     }
2912     pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
2913 
2914     QLIST_FOREACH(block, &ram_list.blocks, next) {
2915         if (!strcmp(block->idstr, new_block->idstr)) {
2916             fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
2917                     new_block->idstr);
2918             abort();
2919         }
2920     }
2921 
2922     if (host) {
2923         new_block->host = host;
2924     } else {
2925         if (mem_path) {
2926 #if defined (__linux__) && !defined(TARGET_S390X)
2927             new_block->host = file_ram_alloc(new_block, size, mem_path);
2928             if (!new_block->host) {
2929                 new_block->host = qemu_vmalloc(size);
2930                 qemu_madvise(new_block->host, size, QEMU_MADV_MERGEABLE);
2931             }
2932 #else
2933             fprintf(stderr, "-mem-path option unsupported\n");
2934             exit(1);
2935 #endif
2936         } else {
2937 #if defined(TARGET_S390X) && defined(CONFIG_KVM)
2938             /* XXX S390 KVM requires the topmost vma of the RAM to be < 256GB */
2939             new_block->host = mmap((void*)0x1000000, size,
2940                                    PROT_EXEC|PROT_READ|PROT_WRITE,
2941                                    MAP_SHARED | MAP_ANONYMOUS, -1, 0);
2942 #else
2943             new_block->host = qemu_vmalloc(size);
2944 #endif
2945 
2946 #ifdef CONFIG_SOLARIS
2947 	/*
2948 	 * XXX For right now, we'll lock down the memory.  This needs to be
2949 	 * revisited if we implement mmu notifiers in the kernel.
2950 	 * Note also that pages are touched in kvm_set_user_memory_region.
2951 	 */
2952 	if (qemu_mlock(new_block->host, size) != 0) {
2953 		fprintf(stderr, "qemu_ram_alloc: Could not lock %ld memory, errno = %d\n",
2954 		      size, errno);
2955 		exit(1);
2956 	}
2957 #endif /*CONFIG_SOLARIS*/
2958 
2959             qemu_madvise(new_block->host, size, QEMU_MADV_MERGEABLE);
2960         }
2961     }
2962 
2963     new_block->offset = find_ram_offset(size);
2964     new_block->length = size;
2965 
2966     QLIST_INSERT_HEAD(&ram_list.blocks, new_block, next);
2967 
2968     ram_list.phys_dirty = qemu_realloc(ram_list.phys_dirty,
2969                                        last_ram_offset() >> TARGET_PAGE_BITS);
2970     memset(ram_list.phys_dirty + (new_block->offset >> TARGET_PAGE_BITS),
2971            0xff, size >> TARGET_PAGE_BITS);
2972 
2973     if (kvm_enabled())
2974         kvm_setup_guest_memory(new_block->host, size);
2975 
2976     return new_block->offset;
2977 }
2978 
qemu_ram_unmap(ram_addr_t addr)2979 void qemu_ram_unmap(ram_addr_t addr)
2980 {
2981     RAMBlock *block;
2982 
2983     QLIST_FOREACH(block, &ram_list.blocks, next) {
2984         if (addr == block->offset) {
2985             QLIST_REMOVE(block, next);
2986             qemu_free(block);
2987             return;
2988         }
2989     }
2990 }
2991 
qemu_ram_alloc(DeviceState * dev,const char * name,ram_addr_t size)2992 ram_addr_t qemu_ram_alloc(DeviceState *dev, const char *name, ram_addr_t size)
2993 {
2994     return qemu_ram_alloc_from_ptr(dev, name, size, NULL);
2995 }
2996 
qemu_ram_free(ram_addr_t addr)2997 void qemu_ram_free(ram_addr_t addr)
2998 {
2999     RAMBlock *block;
3000 
3001     QLIST_FOREACH(block, &ram_list.blocks, next) {
3002         if (addr == block->offset) {
3003             QLIST_REMOVE(block, next);
3004             if (mem_path) {
3005 #if defined (__linux__) && !defined(TARGET_S390X)
3006                 if (block->fd) {
3007                     munmap(block->host, block->length);
3008                     close(block->fd);
3009                 } else {
3010                     qemu_vfree(block->host);
3011                 }
3012 #endif
3013             } else {
3014 #if defined(TARGET_S390X) && defined(CONFIG_KVM)
3015                 munmap(block->host, block->length);
3016 #else
3017                 qemu_vfree(block->host);
3018 #endif
3019             }
3020             qemu_free(block);
3021             return;
3022         }
3023     }
3024 
3025 }
3026 
3027 /* Return a host pointer to ram allocated with qemu_ram_alloc.
3028    With the exception of the softmmu code in this file, this should
3029    only be used for local memory (e.g. video ram) that the device owns,
3030    and knows it isn't going to access beyond the end of the block.
3031 
3032    It should not be used for general purpose DMA.
3033    Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
3034  */
qemu_get_ram_ptr(ram_addr_t addr)3035 void *qemu_get_ram_ptr(ram_addr_t addr)
3036 {
3037     RAMBlock *block;
3038 
3039     QLIST_FOREACH(block, &ram_list.blocks, next) {
3040         if (addr - block->offset < block->length) {
3041             QLIST_REMOVE(block, next);
3042             QLIST_INSERT_HEAD(&ram_list.blocks, block, next);
3043             return block->host + (addr - block->offset);
3044         }
3045     }
3046 
3047     fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
3048     abort();
3049 
3050     return NULL;
3051 }
3052 
3053 /* Return a host pointer to ram allocated with qemu_ram_alloc.
3054  * Same as qemu_get_ram_ptr but avoid reordering ramblocks.
3055  */
qemu_safe_ram_ptr(ram_addr_t addr)3056 void *qemu_safe_ram_ptr(ram_addr_t addr)
3057 {
3058     RAMBlock *block;
3059 
3060     QLIST_FOREACH(block, &ram_list.blocks, next) {
3061         if (addr - block->offset < block->length) {
3062             return block->host + (addr - block->offset);
3063         }
3064     }
3065 
3066     fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
3067     abort();
3068 
3069     return NULL;
3070 }
3071 
qemu_ram_addr_from_host(void * ptr,ram_addr_t * ram_addr)3072 int qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
3073 {
3074     RAMBlock *block;
3075     uint8_t *host = ptr;
3076 
3077     QLIST_FOREACH(block, &ram_list.blocks, next) {
3078         if (host - block->host < block->length) {
3079             *ram_addr = block->offset + (host - block->host);
3080             return 0;
3081         }
3082     }
3083     return -1;
3084 }
3085 
3086 /* Some of the softmmu routines need to translate from a host pointer
3087    (typically a TLB entry) back to a ram offset.  */
qemu_ram_addr_from_host_nofail(void * ptr)3088 ram_addr_t qemu_ram_addr_from_host_nofail(void *ptr)
3089 {
3090     ram_addr_t ram_addr;
3091 
3092     if (qemu_ram_addr_from_host(ptr, &ram_addr)) {
3093         fprintf(stderr, "Bad ram pointer %p\n", ptr);
3094         abort();
3095     }
3096     return ram_addr;
3097 }
3098 
unassigned_mem_readb(void * opaque,target_phys_addr_t addr)3099 static uint32_t unassigned_mem_readb(void *opaque, target_phys_addr_t addr)
3100 {
3101 #ifdef DEBUG_UNASSIGNED
3102     printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
3103 #endif
3104 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
3105     do_unassigned_access(addr, 0, 0, 0, 1);
3106 #endif
3107     return 0;
3108 }
3109 
unassigned_mem_readw(void * opaque,target_phys_addr_t addr)3110 static uint32_t unassigned_mem_readw(void *opaque, target_phys_addr_t addr)
3111 {
3112 #ifdef DEBUG_UNASSIGNED
3113     printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
3114 #endif
3115 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
3116     do_unassigned_access(addr, 0, 0, 0, 2);
3117 #endif
3118     return 0;
3119 }
3120 
unassigned_mem_readl(void * opaque,target_phys_addr_t addr)3121 static uint32_t unassigned_mem_readl(void *opaque, target_phys_addr_t addr)
3122 {
3123 #ifdef DEBUG_UNASSIGNED
3124     printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
3125 #endif
3126 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
3127     do_unassigned_access(addr, 0, 0, 0, 4);
3128 #endif
3129     return 0;
3130 }
3131 
unassigned_mem_writeb(void * opaque,target_phys_addr_t addr,uint32_t val)3132 static void unassigned_mem_writeb(void *opaque, target_phys_addr_t addr, uint32_t val)
3133 {
3134 #ifdef DEBUG_UNASSIGNED
3135     printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
3136 #endif
3137 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
3138     do_unassigned_access(addr, 1, 0, 0, 1);
3139 #endif
3140 }
3141 
unassigned_mem_writew(void * opaque,target_phys_addr_t addr,uint32_t val)3142 static void unassigned_mem_writew(void *opaque, target_phys_addr_t addr, uint32_t val)
3143 {
3144 #ifdef DEBUG_UNASSIGNED
3145     printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
3146 #endif
3147 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
3148     do_unassigned_access(addr, 1, 0, 0, 2);
3149 #endif
3150 }
3151 
unassigned_mem_writel(void * opaque,target_phys_addr_t addr,uint32_t val)3152 static void unassigned_mem_writel(void *opaque, target_phys_addr_t addr, uint32_t val)
3153 {
3154 #ifdef DEBUG_UNASSIGNED
3155     printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
3156 #endif
3157 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
3158     do_unassigned_access(addr, 1, 0, 0, 4);
3159 #endif
3160 }
3161 
3162 static CPUReadMemoryFunc * const unassigned_mem_read[3] = {
3163     unassigned_mem_readb,
3164     unassigned_mem_readw,
3165     unassigned_mem_readl,
3166 };
3167 
3168 static CPUWriteMemoryFunc * const unassigned_mem_write[3] = {
3169     unassigned_mem_writeb,
3170     unassigned_mem_writew,
3171     unassigned_mem_writel,
3172 };
3173 
notdirty_mem_writeb(void * opaque,target_phys_addr_t ram_addr,uint32_t val)3174 static void notdirty_mem_writeb(void *opaque, target_phys_addr_t ram_addr,
3175                                 uint32_t val)
3176 {
3177     int dirty_flags;
3178     dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
3179     if (!(dirty_flags & CODE_DIRTY_FLAG)) {
3180 #if !defined(CONFIG_USER_ONLY)
3181         tb_invalidate_phys_page_fast(ram_addr, 1);
3182         dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
3183 #endif
3184     }
3185     stb_p(qemu_get_ram_ptr(ram_addr), val);
3186     dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
3187     cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
3188     /* we remove the notdirty callback only if the code has been
3189        flushed */
3190     if (dirty_flags == 0xff)
3191         tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
3192 }
3193 
notdirty_mem_writew(void * opaque,target_phys_addr_t ram_addr,uint32_t val)3194 static void notdirty_mem_writew(void *opaque, target_phys_addr_t ram_addr,
3195                                 uint32_t val)
3196 {
3197     int dirty_flags;
3198     dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
3199     if (!(dirty_flags & CODE_DIRTY_FLAG)) {
3200 #if !defined(CONFIG_USER_ONLY)
3201         tb_invalidate_phys_page_fast(ram_addr, 2);
3202         dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
3203 #endif
3204     }
3205     stw_p(qemu_get_ram_ptr(ram_addr), val);
3206     dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
3207     cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
3208     /* we remove the notdirty callback only if the code has been
3209        flushed */
3210     if (dirty_flags == 0xff)
3211         tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
3212 }
3213