Skip to content

Commit

Permalink
mm: add page_check_address_transhuge() helper
Browse files Browse the repository at this point in the history 
page_referenced_one() and page_idle_clear_pte_refs_one() duplicate the
code for looking up pte of a (possibly transhuge) page.  Move this code
to a new helper function, page_check_address_transhuge(), and make the
above mentioned functions use it.

This is just a cleanup, no functional changes are intended.

Signed-off-by: Vladimir Davydov <vdavydov@virtuozzo.com>
Reviewed-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
  • Loading branch information
@torvalds
Vladimir Davydov authored and torvalds committed Jan 16, 2016
1 parent d965432 commit 8749cfe
Show file tree
Hide file tree
Showing 3 changed files with 99 additions and 98 deletions.
19 changes: 19 additions & 0 deletions include/linux/rmap.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -215,6 +215,25 @@ static inline pte_t *page_check_address(struct page *page, struct mm_struct *mm,
return ptep;
}

/*
* Used by idle page tracking to check if a page was referenced via page
* tables.
*/
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
bool page_check_address_transhuge(struct page *page, struct mm_struct *mm,
unsigned long address, pmd_t **pmdp,
pte_t **ptep, spinlock_t **ptlp);
#else
static inline bool page_check_address_transhuge(struct page *page,
struct mm_struct *mm, unsigned long address,
pmd_t **pmdp, pte_t **ptep, spinlock_t **ptlp)
{
*ptep = page_check_address(page, mm, address, ptlp, 0);
*pmdp = NULL;
return !!*ptep;
}
#endif

/*
* Used by swapoff to help locate where page is expected in vma.
*/
Expand Down
63 changes: 9 additions & 54 deletions mm/page_idle.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,71 +55,26 @@ static int page_idle_clear_pte_refs_one(struct page *page,
unsigned long addr, void *arg)
{
struct mm_struct *mm = vma->vm_mm;
spinlock_t *ptl;
pgd_t *pgd;
pud_t *pud;
pmd_t *pmd;
pte_t *pte;
spinlock_t *ptl;
bool referenced = false;

pgd = pgd_offset(mm, addr);
if (!pgd_present(*pgd))
return SWAP_AGAIN;
pud = pud_offset(pgd, addr);
if (!pud_present(*pud))
if (!page_check_address_transhuge(page, mm, addr, &pmd, &pte, &ptl))
return SWAP_AGAIN;
pmd = pmd_offset(pud, addr);

if (pmd_trans_huge(*pmd)) {
ptl = pmd_lock(mm, pmd);
if (!pmd_present(*pmd))
goto unlock_pmd;
if (unlikely(!pmd_trans_huge(*pmd))) {
spin_unlock(ptl);
goto map_pte;
}

if (pmd_page(*pmd) != page)
goto unlock_pmd;

if (pte) {
referenced = ptep_clear_young_notify(vma, addr, pte);
pte_unmap(pte);
} else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) {
referenced = pmdp_clear_young_notify(vma, addr, pmd);
spin_unlock(ptl);
goto found;
unlock_pmd:
spin_unlock(ptl);
return SWAP_AGAIN;
} else {
pmd_t pmde = *pmd;

barrier();
if (!pmd_present(pmde) || pmd_trans_huge(pmde))
return SWAP_AGAIN;

}
map_pte:
pte = pte_offset_map(pmd, addr);
if (!pte_present(*pte)) {
pte_unmap(pte);
return SWAP_AGAIN;
/* unexpected pmd-mapped page? */
WARN_ON_ONCE(1);
}

ptl = pte_lockptr(mm, pmd);
spin_lock(ptl);

if (!pte_present(*pte)) {
pte_unmap_unlock(pte, ptl);
return SWAP_AGAIN;
}

/* THP can be referenced by any subpage */
if (pte_pfn(*pte) - page_to_pfn(page) >= hpage_nr_pages(page)) {
pte_unmap_unlock(pte, ptl);
return SWAP_AGAIN;
}
spin_unlock(ptl);

referenced = ptep_clear_young_notify(vma, addr, pte);
pte_unmap_unlock(pte, ptl);
found:
if (referenced) {
clear_page_idle(page);
/*
Expand Down
115 changes: 71 additions & 44 deletions mm/rmap.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -798,48 +798,44 @@ int page_mapped_in_vma(struct page *page, struct vm_area_struct *vma)
return 1;
}

struct page_referenced_arg {
int mapcount;
int referenced;
unsigned long vm_flags;
struct mem_cgroup *memcg;
};
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
/*
* arg: page_referenced_arg will be passed
* Check that @page is mapped at @address into @mm. In contrast to
* page_check_address(), this function can handle transparent huge pages.
*
* On success returns true with pte mapped and locked. For PMD-mapped
* transparent huge pages *@ptep is set to NULL.
*/
static int page_referenced_one(struct page *page, struct vm_area_struct *vma,
unsigned long address, void *arg)
bool page_check_address_transhuge(struct page *page, struct mm_struct *mm,
unsigned long address, pmd_t **pmdp,
pte_t **ptep, spinlock_t **ptlp)
{
struct mm_struct *mm = vma->vm_mm;
spinlock_t *ptl;
int referenced = 0;
struct page_referenced_arg *pra = arg;
pgd_t *pgd;
pud_t *pud;
pmd_t *pmd;
pte_t *pte;
spinlock_t *ptl;

if (unlikely(PageHuge(page))) {
/* when pud is not present, pte will be NULL */
pte = huge_pte_offset(mm, address);
if (!pte)
return SWAP_AGAIN;
return false;

ptl = huge_pte_lockptr(page_hstate(page), mm, pte);
pmd = NULL;
goto check_pte;
}

pgd = pgd_offset(mm, address);
if (!pgd_present(*pgd))
return SWAP_AGAIN;
return false;
pud = pud_offset(pgd, address);
if (!pud_present(*pud))
return SWAP_AGAIN;
return false;
pmd = pmd_offset(pud, address);

if (pmd_trans_huge(*pmd)) {
int ret = SWAP_AGAIN;

ptl = pmd_lock(mm, pmd);
if (!pmd_present(*pmd))
goto unlock_pmd;
Expand All @@ -851,31 +847,23 @@ static int page_referenced_one(struct page *page, struct vm_area_struct *vma,
if (pmd_page(*pmd) != page)
goto unlock_pmd;

if (vma->vm_flags & VM_LOCKED) {
pra->vm_flags |= VM_LOCKED;
ret = SWAP_FAIL; /* To break the loop */
goto unlock_pmd;
}

if (pmdp_clear_flush_young_notify(vma, address, pmd))
referenced++;
spin_unlock(ptl);
pte = NULL;
goto found;
unlock_pmd:
spin_unlock(ptl);
return ret;
return false;
} else {
pmd_t pmde = *pmd;

barrier();
if (!pmd_present(pmde) || pmd_trans_huge(pmde))
return SWAP_AGAIN;
return false;
}
map_pte:
pte = pte_offset_map(pmd, address);
if (!pte_present(*pte)) {
pte_unmap(pte);
return SWAP_AGAIN;
return false;
}

ptl = pte_lockptr(mm, pmd);
Expand All @@ -884,35 +872,74 @@ static int page_referenced_one(struct page *page, struct vm_area_struct *vma,

if (!pte_present(*pte)) {
pte_unmap_unlock(pte, ptl);
return SWAP_AGAIN;
return false;
}

/* THP can be referenced by any subpage */
if (pte_pfn(*pte) - page_to_pfn(page) >= hpage_nr_pages(page)) {
pte_unmap_unlock(pte, ptl);
return SWAP_AGAIN;
return false;
}
found:
*ptep = pte;
*pmdp = pmd;
*ptlp = ptl;
return true;
}
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */

struct page_referenced_arg {
int mapcount;
int referenced;
unsigned long vm_flags;
struct mem_cgroup *memcg;
};
/*
* arg: page_referenced_arg will be passed
*/
static int page_referenced_one(struct page *page, struct vm_area_struct *vma,
unsigned long address, void *arg)
{
struct mm_struct *mm = vma->vm_mm;
struct page_referenced_arg *pra = arg;
pmd_t *pmd;
pte_t *pte;
spinlock_t *ptl;
int referenced = 0;

if (!page_check_address_transhuge(page, mm, address, &pmd, &pte, &ptl))
return SWAP_AGAIN;

if (vma->vm_flags & VM_LOCKED) {
pte_unmap_unlock(pte, ptl);
if (pte)
pte_unmap(pte);
spin_unlock(ptl);
pra->vm_flags |= VM_LOCKED;
return SWAP_FAIL; /* To break the loop */
}

if (ptep_clear_flush_young_notify(vma, address, pte)) {
/*
* Don't treat a reference through a sequentially read
* mapping as such. If the page has been used in
* another mapping, we will catch it; if this other
* mapping is already gone, the unmap path will have
* set PG_referenced or activated the page.
*/
if (likely(!(vma->vm_flags & VM_SEQ_READ)))
if (pte) {
if (ptep_clear_flush_young_notify(vma, address, pte)) {
/*
* Don't treat a reference through a sequentially read
* mapping as such. If the page has been used in
* another mapping, we will catch it; if this other
* mapping is already gone, the unmap path will have
* set PG_referenced or activated the page.
*/
if (likely(!(vma->vm_flags & VM_SEQ_READ)))
referenced++;
}
pte_unmap(pte);
} else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) {
if (pmdp_clear_flush_young_notify(vma, address, pmd))
referenced++;
} else {
/* unexpected pmd-mapped page? */
WARN_ON_ONCE(1);
}
pte_unmap_unlock(pte, ptl);
spin_unlock(ptl);

found:
if (referenced)
clear_page_idle(page);
if (test_and_clear_page_young(page))
Expand Down

0 comments on commit 8749cfe

Please sign in to comment.