Skip to content

Commit

Permalink
mm/gup: change write parameter to flags in fast walk
Browse files Browse the repository at this point in the history 
In order to support more options in the GUP fast walk, change the write
parameter to flags throughout the call stack.

This patch does not change functionality and passes FOLL_WRITE where write
was previously used.

Link: http://lkml.kernel.org/r/20190328084422.29911-3-ira.weiny@intel.com
Link: http://lkml.kernel.org/r/20190317183438.2057-3-ira.weiny@intel.com
Signed-off-by: Ira Weiny <ira.weiny@intel.com>
Reviewed-by: Dan Williams <dan.j.williams@intel.com>
Cc: Aneesh Kumar K.V <aneesh.kumar@linux.ibm.com>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Borislav Petkov <bp@alien8.de>
Cc: "David S. Miller" <davem@davemloft.net>
Cc: Heiko Carstens <heiko.carstens@de.ibm.com>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: James Hogan <jhogan@kernel.org>
Cc: Jason Gunthorpe <jgg@ziepe.ca>
Cc: John Hubbard <jhubbard@nvidia.com>
Cc: "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com>
Cc: Martin Schwidefsky <schwidefsky@de.ibm.com>
Cc: Michal Hocko <mhocko@kernel.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Ralf Baechle <ralf@linux-mips.org>
Cc: Rich Felker <dalias@libc.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Yoshinori Sato <ysato@users.sourceforge.jp>
Cc: Mike Marshall <hubcap@omnibond.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
  • Loading branch information
@weiny2 @torvalds
weiny2 authored and torvalds committed May 14, 2019
1 parent 932f4a6 commit b798bec
Showing 1 changed file with 26 additions and 26 deletions.
52 changes: 26 additions & 26 deletions mm/gup.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1615,7 +1615,7 @@ static inline struct page *try_get_compound_head(struct page *page, int refs)

#ifdef CONFIG_ARCH_HAS_PTE_SPECIAL
static int gup_pte_range(pmd_t pmd, unsigned long addr, unsigned long end,
int write, struct page **pages, int *nr)
unsigned int flags, struct page **pages, int *nr)
{
struct dev_pagemap *pgmap = NULL;
int nr_start = *nr, ret = 0;
Expand All @@ -1633,7 +1633,7 @@ static int gup_pte_range(pmd_t pmd, unsigned long addr, unsigned long end,
if (pte_protnone(pte))
goto pte_unmap;

if (!pte_access_permitted(pte, write))
if (!pte_access_permitted(pte, flags & FOLL_WRITE))
goto pte_unmap;

if (pte_devmap(pte)) {
Expand Down Expand Up @@ -1685,7 +1685,7 @@ static int gup_pte_range(pmd_t pmd, unsigned long addr, unsigned long end,
* useful to have gup_huge_pmd even if we can't operate on ptes.
*/
static int gup_pte_range(pmd_t pmd, unsigned long addr, unsigned long end,
int write, struct page **pages, int *nr)
unsigned int flags, struct page **pages, int *nr)
{
return 0;
}
Expand Down Expand Up @@ -1768,12 +1768,12 @@ static int __gup_device_huge_pud(pud_t pud, pud_t *pudp, unsigned long addr,
#endif

static int gup_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr,
unsigned long end, int write, struct page **pages, int *nr)
unsigned long end, unsigned int flags, struct page **pages, int *nr)
{
struct page *head, *page;
int refs;

if (!pmd_access_permitted(orig, write))
if (!pmd_access_permitted(orig, flags & FOLL_WRITE))
return 0;

if (pmd_devmap(orig))
Expand Down Expand Up @@ -1806,12 +1806,12 @@ static int gup_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr,
}

static int gup_huge_pud(pud_t orig, pud_t *pudp, unsigned long addr,
unsigned long end, int write, struct page **pages, int *nr)
unsigned long end, unsigned int flags, struct page **pages, int *nr)
{
struct page *head, *page;
int refs;

if (!pud_access_permitted(orig, write))
if (!pud_access_permitted(orig, flags & FOLL_WRITE))
return 0;

if (pud_devmap(orig))
Expand Down Expand Up @@ -1844,13 +1844,13 @@ static int gup_huge_pud(pud_t orig, pud_t *pudp, unsigned long addr,
}

static int gup_huge_pgd(pgd_t orig, pgd_t *pgdp, unsigned long addr,
unsigned long end, int write,
unsigned long end, unsigned int flags,
struct page **pages, int *nr)
{
int refs;
struct page *head, *page;

if (!pgd_access_permitted(orig, write))
if (!pgd_access_permitted(orig, flags & FOLL_WRITE))
return 0;

BUILD_BUG_ON(pgd_devmap(orig));
Expand Down Expand Up @@ -1881,7 +1881,7 @@ static int gup_huge_pgd(pgd_t orig, pgd_t *pgdp, unsigned long addr,
}

static int gup_pmd_range(pud_t pud, unsigned long addr, unsigned long end,
int write, struct page **pages, int *nr)
unsigned int flags, struct page **pages, int *nr)
{
unsigned long next;
pmd_t *pmdp;
Expand All @@ -1904,7 +1904,7 @@ static int gup_pmd_range(pud_t pud, unsigned long addr, unsigned long end,
if (pmd_protnone(pmd))
return 0;

if (!gup_huge_pmd(pmd, pmdp, addr, next, write,
if (!gup_huge_pmd(pmd, pmdp, addr, next, flags,
pages, nr))
return 0;

Expand All @@ -1914,17 +1914,17 @@ static int gup_pmd_range(pud_t pud, unsigned long addr, unsigned long end,
* pmd format and THP pmd format
*/
if (!gup_huge_pd(__hugepd(pmd_val(pmd)), addr,
PMD_SHIFT, next, write, pages, nr))
PMD_SHIFT, next, flags, pages, nr))
return 0;
} else if (!gup_pte_range(pmd, addr, next, write, pages, nr))
} else if (!gup_pte_range(pmd, addr, next, flags, pages, nr))
return 0;
} while (pmdp++, addr = next, addr != end);

return 1;
}

static int gup_pud_range(p4d_t p4d, unsigned long addr, unsigned long end,
int write, struct page **pages, int *nr)
unsigned int flags, struct page **pages, int *nr)
{
unsigned long next;
pud_t *pudp;
Expand All @@ -1937,22 +1937,22 @@ static int gup_pud_range(p4d_t p4d, unsigned long addr, unsigned long end,
if (pud_none(pud))
return 0;
if (unlikely(pud_huge(pud))) {
if (!gup_huge_pud(pud, pudp, addr, next, write,
if (!gup_huge_pud(pud, pudp, addr, next, flags,
pages, nr))
return 0;
} else if (unlikely(is_hugepd(__hugepd(pud_val(pud))))) {
if (!gup_huge_pd(__hugepd(pud_val(pud)), addr,
PUD_SHIFT, next, write, pages, nr))
PUD_SHIFT, next, flags, pages, nr))
return 0;
} else if (!gup_pmd_range(pud, addr, next, write, pages, nr))
} else if (!gup_pmd_range(pud, addr, next, flags, pages, nr))
return 0;
} while (pudp++, addr = next, addr != end);

return 1;
}

static int gup_p4d_range(pgd_t pgd, unsigned long addr, unsigned long end,
int write, struct page **pages, int *nr)
unsigned int flags, struct page **pages, int *nr)
{
unsigned long next;
p4d_t *p4dp;
Expand All @@ -1967,17 +1967,17 @@ static int gup_p4d_range(pgd_t pgd, unsigned long addr, unsigned long end,
BUILD_BUG_ON(p4d_huge(p4d));
if (unlikely(is_hugepd(__hugepd(p4d_val(p4d))))) {
if (!gup_huge_pd(__hugepd(p4d_val(p4d)), addr,
P4D_SHIFT, next, write, pages, nr))
P4D_SHIFT, next, flags, pages, nr))
return 0;
} else if (!gup_pud_range(p4d, addr, next, write, pages, nr))
} else if (!gup_pud_range(p4d, addr, next, flags, pages, nr))
return 0;
} while (p4dp++, addr = next, addr != end);

return 1;
}

static void gup_pgd_range(unsigned long addr, unsigned long end,
int write, struct page **pages, int *nr)
unsigned int flags, struct page **pages, int *nr)
{
unsigned long next;
pgd_t *pgdp;
Expand All @@ -1990,14 +1990,14 @@ static void gup_pgd_range(unsigned long addr, unsigned long end,
if (pgd_none(pgd))
return;
if (unlikely(pgd_huge(pgd))) {
if (!gup_huge_pgd(pgd, pgdp, addr, next, write,
if (!gup_huge_pgd(pgd, pgdp, addr, next, flags,
pages, nr))
return;
} else if (unlikely(is_hugepd(__hugepd(pgd_val(pgd))))) {
if (!gup_huge_pd(__hugepd(pgd_val(pgd)), addr,
PGDIR_SHIFT, next, write, pages, nr))
PGDIR_SHIFT, next, flags, pages, nr))
return;
} else if (!gup_p4d_range(pgd, addr, next, write, pages, nr))
} else if (!gup_p4d_range(pgd, addr, next, flags, pages, nr))
return;
} while (pgdp++, addr = next, addr != end);
}
Expand Down Expand Up @@ -2051,7 +2051,7 @@ int __get_user_pages_fast(unsigned long start, int nr_pages, int write,

if (gup_fast_permitted(start, nr_pages)) {
local_irq_save(flags);
gup_pgd_range(start, end, write, pages, &nr);
gup_pgd_range(start, end, write ? FOLL_WRITE : 0, pages, &nr);
local_irq_restore(flags);
}

Expand Down Expand Up @@ -2093,7 +2093,7 @@ int get_user_pages_fast(unsigned long start, int nr_pages, int write,

if (gup_fast_permitted(start, nr_pages)) {
local_irq_disable();
gup_pgd_range(addr, end, write, pages, &nr);
gup_pgd_range(addr, end, write ? FOLL_WRITE : 0, pages, &nr);
local_irq_enable();
ret = nr;
}
Expand Down

0 comments on commit b798bec

Please sign in to comment.