lineage_kernel_xcoverpro/mm/hpa.c

/*
 * linux/mm/hpa.c
 *
 * Copyright (C) 2015 Samsung Electronics, Inc. All Rights Reserved.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version
 * 2 of the License, or (at your option) any later version.

 * Does best efforts to allocate required high-order pages.
 */

#include <linux/list.h>
#include <linux/bootmem.h>
#include <linux/memblock.h>
#include <linux/mm.h>
#include <linux/mm_types.h>
#include <linux/mmzone.h>
#include <linux/migrate.h>
#include <linux/memcontrol.h>
#include <linux/page-isolation.h>
#include <linux/mm_inline.h>
#include <linux/swap.h>
#include <linux/scatterlist.h>
#include <linux/debugfs.h>
#include <linux/vmalloc.h>
#include <linux/device.h>
#include <linux/oom.h>
#include <linux/sched/task.h>
#include <linux/sched/mm.h>

#include "internal.h"

#define MAX_SCAN_TRY		(2)

static unsigned long start_pfn, end_pfn;
static unsigned long cached_scan_pfn;

#define HPA_MIN_OOMADJ	100

static bool oom_unkillable_task(struct task_struct *p)
{
	if (is_global_init(p))
		return true;
	if (p->flags & PF_KTHREAD)
		return true;
	return false;
}

static bool oom_skip_task(struct task_struct *p, int selected_adj)
{
	if (same_thread_group(p, current))
		return true;
	if (p->signal->oom_score_adj <= HPA_MIN_OOMADJ)
		return true;
	if ((p->signal->oom_score_adj < selected_adj) &&
	    (selected_adj <= OOM_SCORE_ADJ_MAX))
		return true;
	if (test_bit(MMF_OOM_SKIP, &p->mm->flags))
		return true;
	if (in_vfork(p))
		return true;
	if (p->state & TASK_UNINTERRUPTIBLE)
		return true;
	return false;
}

static int hpa_killer(void)
{
	struct task_struct *tsk, *p;
	struct task_struct *selected = NULL;
	unsigned long selected_tasksize = 0;
	int selected_adj = OOM_SCORE_ADJ_MAX + 1;

	rcu_read_lock();
	for_each_process(tsk) {
		int tasksize;
		int current_adj;

		if (oom_unkillable_task(tsk))
			continue;

		p = find_lock_task_mm(tsk);
		if (!p)
			continue;

		if (oom_skip_task(p, selected_adj)) {
			task_unlock(p);
			continue;
		}

		tasksize = get_mm_rss(p->mm);
		tasksize += get_mm_counter(p->mm, MM_SWAPENTS);
		tasksize += atomic_long_read(&p->mm->nr_ptes);
		tasksize += mm_nr_pmds(p->mm);
		current_adj = p->signal->oom_score_adj;

		task_unlock(p);

		if (selected &&
		    (current_adj == selected_adj) &&
		    (tasksize <= selected_tasksize))
			continue;

		if (selected)
			put_task_struct(selected);

		selected = p;
		selected_tasksize = tasksize;
		selected_adj = current_adj;
		get_task_struct(selected);
	}
	rcu_read_unlock();

	if (!selected) {
		pr_info("HPA: no killable task\n");
		return -ESRCH;
	}

	pr_info("HPA: Killing '%s' (%d), adj %hd to free %lukB\n",
		selected->comm, task_pid_nr(selected), selected_adj,
		selected_tasksize * (PAGE_SIZE / SZ_1K));

	do_send_sig_info(SIGKILL, SEND_SIG_FORCED, selected, true);

	put_task_struct(selected);

	return 0;
}

static bool is_movable_chunk(unsigned long pfn, unsigned int order)
{
	struct page *page = pfn_to_page(pfn);
	struct page *page_end = pfn_to_page(pfn + (1 << order));

	while (page != page_end) {
		if (PageCompound(page) || PageReserved(page))
			return false;
		if (!PageLRU(page) && !__PageMovable(page))
			return false;

		page += PageBuddy(page) ? 1 << page_order(page) : 1;
	}

	return true;
}

static int get_exception_of_page(phys_addr_t phys,
				 phys_addr_t exception_areas[][2],
				 int nr_exception)
{
	int i;

	for (i = 0; i < nr_exception; i++)
		if ((exception_areas[i][0] <= phys) &&
		    (phys <= exception_areas[i][1]))
			return i;
	return -1;
}

static inline void expand(struct zone *zone, struct page *page,
			  int low, int high, struct free_area *area,
			  int migratetype)
{
	unsigned long size = 1 << high;

	while (high > low) {
		area--;
		high--;
		size >>= 1;

		list_add(&page[size].lru, &area->free_list[migratetype]);
		area->nr_free++;
		set_page_private(&page[size], high);
		__SetPageBuddy(&page[size]);
	}
}

static struct page *alloc_freepage_one(struct zone *zone, unsigned int order,
				       phys_addr_t exception_areas[][2],
				       int nr_exception)
{
	unsigned int current_order;
	struct free_area *area;
	struct page *page;
	int mt;

	for (mt = MIGRATE_UNMOVABLE; mt < MIGRATE_PCPTYPES; ++mt) {
		for (current_order = order;
		     current_order < MAX_ORDER; ++current_order) {
			area = &(zone->free_area[current_order]);

			list_for_each_entry(page, &area->free_list[mt], lru) {
				if (get_exception_of_page(page_to_phys(page),
							  exception_areas,
							  nr_exception) >= 0)
					continue;

				list_del(&page->lru);

				__ClearPageBuddy(page);
				set_page_private(page, 0);
				area->nr_free--;
				expand(zone, page, order,
				       current_order, area, mt);
				set_pcppage_migratetype(page, mt);

				return page;
			}
		}
	}

	return NULL;
}

static int alloc_freepages_range(struct zone *zone, unsigned int order,
				 struct page **pages, int required,
				 phys_addr_t exception_areas[][2],
				 int nr_exception)

{
	unsigned long wmark;
	unsigned long flags;
	struct page *page;
	int count = 0;

	spin_lock_irqsave(&zone->lock, flags);

	while (required > count) {
		wmark = min_wmark_pages(zone) + (1 << order);
		if (!zone_watermark_ok(zone, order, wmark, 0, 0))
			goto wmark_fail;

		page = alloc_freepage_one(zone, order, exception_areas,
					  nr_exception);
		if (!page)
			break;

		post_alloc_hook(page, order, GFP_KERNEL);
		__mod_zone_page_state(zone, NR_FREE_PAGES, -(1 << order));
		pages[count++] = page;
		__count_zid_vm_events(PGALLOC, page_zonenum(page), 1 << order);
	}

wmark_fail:
	spin_unlock_irqrestore(&zone->lock, flags);

	return count;
}

static void prep_highorder_pages(unsigned long base_pfn, int order)
{
	int nr_pages = 1 << order;
	unsigned long pfn;

	for (pfn = base_pfn + 1; pfn < base_pfn + nr_pages; pfn++)
		set_page_count(pfn_to_page(pfn), 0);
}

/**
 * alloc_pages_highorder_except() - allocate large order pages
 * @order:           required page order
 * @pages:           array to store allocated @order order pages
 * @nents:           number of @order order pages
 * @exception_areas: memory areas that should not include pages in @pages
 * @nr_exception:    number of memory areas in @exception_areas
 *
 * Returns 0 on allocation success. -error otherwise.
 *
 * Allocates @nents pages of @order << PAGE_SHIFT number of consecutive pages
 * and store the page descriptors of the allocated pages to @pages. Every page
 * in @pages should also be aligned by @order << PAGE_SHIFT.
 *
 * If @nr_exception is larger than 0, alloc_page_highorder_except() does not
 * allocate pages in the areas described in @exception_areas. @exception_areas
 * is an array of array with two elements: The first element is the start
 * address of an area and the last element is the end address. The end address
 * is the last byte address in the area, that is "[start address] + [size] - 1".
 */
int alloc_pages_highorder_except(int order, struct page **pages, int nents,
				 phys_addr_t exception_areas[][2],
				 int nr_exception)
{
	struct zone *zone;
	unsigned int nr_pages = 1 << order;
	unsigned long total_scanned = 0;
	unsigned long pfn, tmp;
	int remained = nents;
	int ret;
	int retry_count = 0;
	int allocated;

retry:
	for_each_zone(zone) {
		if (zone->spanned_pages == 0)
			continue;

		allocated = alloc_freepages_range(zone, order,
					pages + nents - remained, remained,
					exception_areas, nr_exception);
		remained -= allocated;

		if (remained == 0)
			return 0;
	}

	migrate_prep();

	for (pfn = ALIGN(cached_scan_pfn, nr_pages);
			(total_scanned < (end_pfn - start_pfn) * MAX_SCAN_TRY)
			&& (remained > 0);
			pfn += nr_pages, total_scanned += nr_pages) {
		int mt;

		if (pfn + nr_pages > end_pfn) {
			pfn = start_pfn;
			continue;
		}

		/* pfn validation check in the range */
		tmp = pfn;
		do {
			if (!pfn_valid(tmp))
				break;
		} while (++tmp < (pfn + nr_pages));

		if (tmp < (pfn + nr_pages))
			continue;

		mt = get_pageblock_migratetype(pfn_to_page(pfn));
		/*
		 * CMA pages should not be reclaimed.
		 * Isolated page blocks should not be tried again because it
		 * causes isolated page block remained in isolated state
		 * forever.
		 */
		if (is_migrate_cma(mt) || is_migrate_isolate(mt)) {
			/* nr_pages is added before next iteration */
			pfn = ALIGN(pfn + 1, pageblock_nr_pages) - nr_pages;
			continue;
		}

		ret = get_exception_of_page(pfn << PAGE_SHIFT,
					    exception_areas, nr_exception);
		if (ret >= 0) {
			pfn = (exception_areas[ret][1] + 1) >> PAGE_SHIFT;
			pfn -= nr_pages;
			continue;
		}

		if (!is_movable_chunk(pfn, order))
			continue;

		ret = alloc_contig_range_fast(pfn, pfn + nr_pages, mt);
		if (ret == 0)
			prep_highorder_pages(pfn, order);
		else
			continue;

		pages[nents - remained] = pfn_to_page(pfn);
		remained--;
	}

	/* save latest scanned pfn */
	cached_scan_pfn = pfn;

	if (remained) {
		int i;

		drop_slab();
		count_vm_event(DROP_SLAB);
		ret = hpa_killer();
		if (ret == 0) {
			total_scanned = 0;
			pr_info("HPA: drop_slab and killer retry %d count\n",
				retry_count++);
			goto retry;
		}

		for (i = 0; i < (nents - remained); i++)
			__free_pages(pages[i], order);

		pr_info("%s: remained=%d / %d, not enough memory in order %d\n",
				 __func__, remained, nents, order);

		ret = -ENOMEM;
	}

	return ret;
}

int free_pages_highorder(int order, struct page **pages, int nents)
{
	int i;

	for (i = 0; i < nents; i++)
		__free_pages(pages[i], order);

	return 0;
}

static int __init init_highorder_pages_allocator(void)
{
	struct zone *zone;

	for_each_zone(zone) {
		if (zone->spanned_pages == 0)
			continue;
		if (zone_idx(zone) == ZONE_MOVABLE) {
			start_pfn = zone->zone_start_pfn;
			end_pfn = start_pfn + zone->present_pages;
		}
	}

	if (!start_pfn) {
		start_pfn = __phys_to_pfn(memblock_start_of_DRAM());
		end_pfn = max_pfn;
	}

	cached_scan_pfn = start_pfn;

	return 0;
}
late_initcall(init_highorder_pages_allocator);